2026-02-20T20:57:48.700 INFO:root:teuthology version: 1.2.4.dev9+gfa17720d0 2026-02-20T20:57:48.714 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-20T20:57:48.796 INFO:teuthology.run:Config: archive_path: /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61415 branch: wip-pdonnell-testing-20260219.182737-tentacle description: rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-snappy supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-workunit-fast-diff} email: yweinste@redhat.com first_in_suite: false flavor: default job_id: '61415' kernel: branch: distro kdb: true sha1: distro last_in_suite: false machine_type: trial meta: - desc: run two ceph clusters and install rbd-mirror - desc: configure the permissions for client.mirror - desc: configure the permissions for client.mirror - desc: 2 ceph clusters with 1 mon and 3 osds each - desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon in snapshot mode name: yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial no_nested_subset: false os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-pdonnell-testing-20260219.182737-tentacle ceph: conf: client: debug rbd: 20 debug rbd_mirror: 15 log to stderr: false rbd default features: 125 client.mirror.0: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.1: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.2: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.3: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.4: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.5: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.6: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid global: mon client directed command retry: 5 mon warn on pool no app: false ms inject socket failures: 5000 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: bluestore block size: 96636764160 bluestore compression algorithm: snappy bluestore compression mode: aggressive bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(OSD_SLOW_PING_TIME sha1: f7870454149f22574086dd2ff55d050585136531 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: f7870454149f22574086dd2ff55d050585136531 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: wip-pdonnell-testing-20260219.182737-tentacle sha1: f7870454149f22574086dd2ff55d050585136531 owner: scheduled_yuriw@soko04.front.sepia.ceph.com priority: 69 repo: https://github.com/ceph/ceph-ci.git roles: - - cluster1.mon.a - cluster1.mgr.x - cluster2.mgr.x - cluster1.osd.0 - cluster1.osd.1 - cluster1.osd.2 - cluster1.client.0 - cluster2.client.0 - - cluster2.mon.a - cluster2.osd.0 - cluster2.osd.1 - cluster2.osd.2 - cluster1.client.mirror - cluster1.client.mirror.0 - cluster1.client.mirror.1 - cluster1.client.mirror.2 - cluster1.client.mirror.3 - cluster1.client.mirror.4 - cluster1.client.mirror.5 - cluster1.client.mirror.6 - cluster2.client.mirror - cluster2.client.mirror.0 - cluster2.client.mirror.1 - cluster2.client.mirror.2 - cluster2.client.mirror.3 - cluster2.client.mirror.4 - cluster2.client.mirror.5 - cluster2.client.mirror.6 seed: 7161 sha1: f7870454149f22574086dd2ff55d050585136531 sleep_before_teardown: 0 subset: 111/120000 suite: rbd suite_branch: wip-pdonnell-testing-20260219.182737-tentacle suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_f7870454149f22574086dd2ff55d050585136531/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci.git suite_sha1: f7870454149f22574086dd2ff55d050585136531 targets: trial196.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNXSz/oleXj4WCyNHwLu/9imobgWwskZJuDoZU2v2KihsuN8A40ibPZsGroPVka9yfVxYbjvvfMQvHwms8hWr/I= trial199.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPM+ZdBRamPjFyytTKVlvfP6NMc801M2vKxXYLe18YXGDgXMyw4c98dOBWD/E0Bn+Cp7UqFHFWoxeALuafPFuco= tasks: - install: extra_packages: - rbd-mirror - ceph: cluster: cluster1 - ceph: cluster: cluster2 - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd' - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - workunit: clients: cluster1.client.mirror: - rbd/rbd_mirror.sh env: CEPH_ARGS: '' RBD_IMAGE_FEATURES: layering,exclusive-lock,object-map,fast-diff RBD_MIRROR_CONFIG_KEY: '1' RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_MODE: snapshot RBD_MIRROR_USE_EXISTING_CLUSTER: '1' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: fa17720d0088c3ac28e473468bfc79eeaff5cd38 timestamp: 2026-02-20_16:15:01 tube: trial user: yuriw verbose: true worker_log: /home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.1540699 2026-02-20T20:57:48.797 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_f7870454149f22574086dd2ff55d050585136531/qa; will attempt to use it 2026-02-20T20:57:48.797 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_f7870454149f22574086dd2ff55d050585136531/qa/tasks 2026-02-20T20:57:48.797 INFO:teuthology.run_tasks:Running task internal.check_packages... 2026-02-20T20:57:48.799 INFO:teuthology.task.internal:Checking packages... 2026-02-20T20:57:48.799 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'f7870454149f22574086dd2ff55d050585136531' 2026-02-20T20:57:48.799 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2026-02-20T20:57:48.799 INFO:teuthology.packaging:ref: None 2026-02-20T20:57:48.799 INFO:teuthology.packaging:tag: None 2026-02-20T20:57:48.799 INFO:teuthology.packaging:branch: wip-pdonnell-testing-20260219.182737-tentacle 2026-02-20T20:57:48.799 INFO:teuthology.packaging:sha1: f7870454149f22574086dd2ff55d050585136531 2026-02-20T20:57:48.799 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-pdonnell-testing-20260219.182737-tentacle 2026-02-20T20:57:48.920 INFO:teuthology.task.internal:Found packages for ceph version 20.2.0-677.gf7870454 2026-02-20T20:57:48.921 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2026-02-20T20:57:48.931 INFO:teuthology.task.internal:no buildpackages task found 2026-02-20T20:57:48.931 INFO:teuthology.run_tasks:Running task internal.save_config... 2026-02-20T20:57:48.944 INFO:teuthology.task.internal:Saving configuration 2026-02-20T20:57:48.958 INFO:teuthology.run_tasks:Running task internal.check_lock... 2026-02-20T20:57:48.970 INFO:teuthology.task.internal.check_lock:Checking locks... 2026-02-20T20:57:49.046 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial196.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61415', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-20 20:54:21.049542', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNXSz/oleXj4WCyNHwLu/9imobgWwskZJuDoZU2v2KihsuN8A40ibPZsGroPVka9yfVxYbjvvfMQvHwms8hWr/I='} 2026-02-20T20:57:49.103 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial199.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61415', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-20 20:54:21.050326', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPM+ZdBRamPjFyytTKVlvfP6NMc801M2vKxXYLe18YXGDgXMyw4c98dOBWD/E0Bn+Cp7UqFHFWoxeALuafPFuco='} 2026-02-20T20:57:49.103 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2026-02-20T20:57:49.115 INFO:teuthology.task.internal:roles: ubuntu@trial196.front.sepia.ceph.com - ['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'] 2026-02-20T20:57:49.115 INFO:teuthology.task.internal:roles: ubuntu@trial199.front.sepia.ceph.com - ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6'] 2026-02-20T20:57:49.115 INFO:teuthology.run_tasks:Running task console_log... 2026-02-20T20:57:49.247 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x728a22bd4cc0>, signals=[15]) 2026-02-20T20:57:49.247 INFO:teuthology.run_tasks:Running task internal.connect... 2026-02-20T20:57:49.259 INFO:teuthology.task.internal:Opening connections... 2026-02-20T20:57:49.259 DEBUG:teuthology.task.internal:connecting to ubuntu@trial196.front.sepia.ceph.com 2026-02-20T20:57:49.259 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial196.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-20T20:57:49.320 DEBUG:teuthology.task.internal:connecting to ubuntu@trial199.front.sepia.ceph.com 2026-02-20T20:57:49.321 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial199.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-20T20:57:49.384 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2026-02-20T20:57:49.395 DEBUG:teuthology.orchestra.run.trial196:> uname -m 2026-02-20T20:57:49.411 INFO:teuthology.orchestra.run.trial196.stdout:x86_64 2026-02-20T20:57:49.412 DEBUG:teuthology.orchestra.run.trial196:> cat /etc/os-release 2026-02-20T20:57:49.467 INFO:teuthology.orchestra.run.trial196.stdout:NAME="CentOS Stream" 2026-02-20T20:57:49.468 INFO:teuthology.orchestra.run.trial196.stdout:VERSION="9" 2026-02-20T20:57:49.468 INFO:teuthology.orchestra.run.trial196.stdout:ID="centos" 2026-02-20T20:57:49.468 INFO:teuthology.orchestra.run.trial196.stdout:ID_LIKE="rhel fedora" 2026-02-20T20:57:49.468 INFO:teuthology.orchestra.run.trial196.stdout:VERSION_ID="9" 2026-02-20T20:57:49.468 INFO:teuthology.orchestra.run.trial196.stdout:PLATFORM_ID="platform:el9" 2026-02-20T20:57:49.468 INFO:teuthology.orchestra.run.trial196.stdout:PRETTY_NAME="CentOS Stream 9" 2026-02-20T20:57:49.468 INFO:teuthology.orchestra.run.trial196.stdout:ANSI_COLOR="0;31" 2026-02-20T20:57:49.468 INFO:teuthology.orchestra.run.trial196.stdout:LOGO="fedora-logo-icon" 2026-02-20T20:57:49.468 INFO:teuthology.orchestra.run.trial196.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-02-20T20:57:49.468 INFO:teuthology.orchestra.run.trial196.stdout:HOME_URL="https://centos.org/" 2026-02-20T20:57:49.468 INFO:teuthology.orchestra.run.trial196.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-02-20T20:57:49.468 INFO:teuthology.orchestra.run.trial196.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-02-20T20:57:49.468 INFO:teuthology.orchestra.run.trial196.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-02-20T20:57:49.468 INFO:teuthology.lock.ops:Updating trial196.front.sepia.ceph.com on lock server 2026-02-20T20:57:49.539 DEBUG:teuthology.orchestra.run.trial199:> uname -m 2026-02-20T20:57:49.555 INFO:teuthology.orchestra.run.trial199.stdout:x86_64 2026-02-20T20:57:49.555 DEBUG:teuthology.orchestra.run.trial199:> cat /etc/os-release 2026-02-20T20:57:49.610 INFO:teuthology.orchestra.run.trial199.stdout:NAME="CentOS Stream" 2026-02-20T20:57:49.610 INFO:teuthology.orchestra.run.trial199.stdout:VERSION="9" 2026-02-20T20:57:49.610 INFO:teuthology.orchestra.run.trial199.stdout:ID="centos" 2026-02-20T20:57:49.610 INFO:teuthology.orchestra.run.trial199.stdout:ID_LIKE="rhel fedora" 2026-02-20T20:57:49.610 INFO:teuthology.orchestra.run.trial199.stdout:VERSION_ID="9" 2026-02-20T20:57:49.610 INFO:teuthology.orchestra.run.trial199.stdout:PLATFORM_ID="platform:el9" 2026-02-20T20:57:49.610 INFO:teuthology.orchestra.run.trial199.stdout:PRETTY_NAME="CentOS Stream 9" 2026-02-20T20:57:49.610 INFO:teuthology.orchestra.run.trial199.stdout:ANSI_COLOR="0;31" 2026-02-20T20:57:49.610 INFO:teuthology.orchestra.run.trial199.stdout:LOGO="fedora-logo-icon" 2026-02-20T20:57:49.610 INFO:teuthology.orchestra.run.trial199.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-02-20T20:57:49.610 INFO:teuthology.orchestra.run.trial199.stdout:HOME_URL="https://centos.org/" 2026-02-20T20:57:49.610 INFO:teuthology.orchestra.run.trial199.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-02-20T20:57:49.610 INFO:teuthology.orchestra.run.trial199.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-02-20T20:57:49.610 INFO:teuthology.orchestra.run.trial199.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-02-20T20:57:49.611 INFO:teuthology.lock.ops:Updating trial199.front.sepia.ceph.com on lock server 2026-02-20T20:57:49.686 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2026-02-20T20:57:49.699 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2026-02-20T20:57:49.709 INFO:teuthology.task.internal:Checking for old test directory... 2026-02-20T20:57:49.709 DEBUG:teuthology.orchestra.run.trial196:> test '!' -e /home/ubuntu/cephtest 2026-02-20T20:57:49.711 DEBUG:teuthology.orchestra.run.trial199:> test '!' -e /home/ubuntu/cephtest 2026-02-20T20:57:49.726 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2026-02-20T20:57:49.736 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2026-02-20T20:57:49.737 DEBUG:teuthology.orchestra.run.trial196:> test -z $(ls -A /var/lib/ceph) 2026-02-20T20:57:49.767 DEBUG:teuthology.orchestra.run.trial199:> test -z $(ls -A /var/lib/ceph) 2026-02-20T20:57:49.783 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2026-02-20T20:57:49.947 INFO:teuthology.run_tasks:Running task kernel... 2026-02-20T20:57:49.966 INFO:teuthology.task.kernel:normalize config orig: {'branch': 'distro', 'kdb': True, 'sha1': 'distro'} 2026-02-20T20:57:49.966 INFO:teuthology.task.kernel:config {'cluster1.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'cluster2.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, timeout 300 2026-02-20T20:57:49.967 DEBUG:teuthology.orchestra.run.trial196:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-20T20:57:49.967 DEBUG:teuthology.orchestra.run.trial199:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-20T20:57:49.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T20:57:49.982 DEBUG:teuthology.orchestra.run.trial196:> uname -r 2026-02-20T20:57:49.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T20:57:49.982 DEBUG:teuthology.orchestra.run.trial199:> uname -r 2026-02-20T20:57:50.037 INFO:teuthology.orchestra.run.trial196.stdout:5.14.0-681.el9.x86_64 2026-02-20T20:57:50.037 INFO:teuthology.task.kernel:Running kernel on trial196: 5.14.0-681.el9.x86_64 2026-02-20T20:57:50.037 DEBUG:teuthology.orchestra.run.trial196:> sudo yum install -y kernel 2026-02-20T20:57:50.038 INFO:teuthology.orchestra.run.trial199.stdout:5.14.0-681.el9.x86_64 2026-02-20T20:57:50.038 INFO:teuthology.task.kernel:Running kernel on trial199: 5.14.0-681.el9.x86_64 2026-02-20T20:57:50.038 DEBUG:teuthology.orchestra.run.trial199:> sudo yum install -y kernel 2026-02-20T20:57:51.962 INFO:teuthology.orchestra.run.trial196.stdout:CentOS Stream 9 - BaseOS 5.7 MB/s | 8.9 MB 00:01 2026-02-20T20:57:52.101 INFO:teuthology.orchestra.run.trial199.stdout:CentOS Stream 9 - BaseOS 5.3 MB/s | 8.9 MB 00:01 2026-02-20T20:57:55.091 INFO:teuthology.orchestra.run.trial199.stdout:CentOS Stream 9 - AppStream 11 MB/s | 27 MB 00:02 2026-02-20T20:57:55.644 INFO:teuthology.orchestra.run.trial196.stdout:CentOS Stream 9 - AppStream 8.5 MB/s | 27 MB 00:03 2026-02-20T20:57:58.676 INFO:teuthology.orchestra.run.trial199.stdout:CentOS Stream 9 - CRB 6.3 MB/s | 7.8 MB 00:01 2026-02-20T20:58:00.282 INFO:teuthology.orchestra.run.trial199.stdout:CentOS Stream 9 - Extras packages 23 kB/s | 20 kB 00:00 2026-02-20T20:58:01.125 INFO:teuthology.orchestra.run.trial199.stdout:Extra Packages for Enterprise Linux 26 MB/s | 20 MB 00:00 2026-02-20T20:58:04.293 INFO:teuthology.orchestra.run.trial199.stdout:lab-extras 1.7 MB/s | 50 kB 00:00 2026-02-20T20:58:05.304 INFO:teuthology.orchestra.run.trial199.stdout:Package kernel-5.14.0-645.el9.x86_64 is already installed. 2026-02-20T20:58:05.304 INFO:teuthology.orchestra.run.trial199.stdout:Package kernel-5.14.0-681.el9.x86_64 is already installed. 2026-02-20T20:58:05.323 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T20:58:05.323 INFO:teuthology.orchestra.run.trial199.stdout:Nothing to do. 2026-02-20T20:58:05.324 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T20:58:05.348 DEBUG:teuthology.orchestra.run.trial199:> echo no | sudo yum reinstall kernel || true 2026-02-20T20:58:05.646 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:01 ago on Fri 20 Feb 2026 08:58:04 PM UTC. 2026-02-20T20:58:05.726 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T20:58:05.727 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T20:58:05.728 INFO:teuthology.orchestra.run.trial199.stdout: Package Architecture Version Repository Size 2026-02-20T20:58:05.728 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T20:58:05.728 INFO:teuthology.orchestra.run.trial199.stdout:Reinstalling: 2026-02-20T20:58:05.728 INFO:teuthology.orchestra.run.trial199.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-20T20:58:05.728 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T20:58:05.728 INFO:teuthology.orchestra.run.trial199.stdout:Transaction Summary 2026-02-20T20:58:05.728 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T20:58:05.728 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T20:58:05.729 INFO:teuthology.orchestra.run.trial199.stdout:Total download size: 803 k 2026-02-20T20:58:05.729 INFO:teuthology.orchestra.run.trial199.stdout:Installed size: 0 2026-02-20T20:58:05.729 INFO:teuthology.orchestra.run.trial199.stderr:Operation aborted. 2026-02-20T20:58:05.746 INFO:teuthology.orchestra.run.trial199.stdout:Is this ok [y/N]: 2026-02-20T20:58:05.747 DEBUG:teuthology.orchestra.run.trial199:> sudo yum reinstall -y kernel || true 2026-02-20T20:58:06.048 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:02 ago on Fri 20 Feb 2026 08:58:04 PM UTC. 2026-02-20T20:58:06.129 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T20:58:06.129 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T20:58:06.129 INFO:teuthology.orchestra.run.trial199.stdout: Package Architecture Version Repository Size 2026-02-20T20:58:06.129 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T20:58:06.129 INFO:teuthology.orchestra.run.trial199.stdout:Reinstalling: 2026-02-20T20:58:06.129 INFO:teuthology.orchestra.run.trial199.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-20T20:58:06.129 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T20:58:06.129 INFO:teuthology.orchestra.run.trial199.stdout:Transaction Summary 2026-02-20T20:58:06.129 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T20:58:06.129 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T20:58:06.129 INFO:teuthology.orchestra.run.trial199.stdout:Total download size: 803 k 2026-02-20T20:58:06.129 INFO:teuthology.orchestra.run.trial199.stdout:Installed size: 0 2026-02-20T20:58:06.129 INFO:teuthology.orchestra.run.trial199.stdout:Downloading Packages: 2026-02-20T20:58:06.327 INFO:teuthology.orchestra.run.trial199.stdout:kernel-5.14.0-681.el9.x86_64.rpm 9.5 MB/s | 803 kB 00:00 2026-02-20T20:58:06.327 INFO:teuthology.orchestra.run.trial199.stdout:-------------------------------------------------------------------------------- 2026-02-20T20:58:06.327 INFO:teuthology.orchestra.run.trial199.stdout:Total 4.0 MB/s | 803 kB 00:00 2026-02-20T20:58:06.337 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction check 2026-02-20T20:58:06.370 INFO:teuthology.orchestra.run.trial199.stdout:Transaction check succeeded. 2026-02-20T20:58:06.370 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction test 2026-02-20T20:58:06.375 INFO:teuthology.orchestra.run.trial199.stdout:Transaction test succeeded. 2026-02-20T20:58:06.375 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction 2026-02-20T20:58:06.399 INFO:teuthology.orchestra.run.trial199.stdout: Preparing : 1/1 2026-02-20T20:58:06.403 INFO:teuthology.orchestra.run.trial199.stdout: Reinstalling : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-20T20:58:06.403 INFO:teuthology.orchestra.run.trial199.stdout: Cleanup : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-20T20:58:06.510 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: kernel-5.14.0-681.el9.x86_64 2/2 2026-02-20T20:58:06.511 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-20T20:58:06.562 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-20T20:58:06.562 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T20:58:06.562 INFO:teuthology.orchestra.run.trial199.stdout:Reinstalled: 2026-02-20T20:58:06.562 INFO:teuthology.orchestra.run.trial199.stdout: kernel-5.14.0-681.el9.x86_64 2026-02-20T20:58:06.562 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T20:58:06.562 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T20:58:06.590 DEBUG:teuthology.orchestra.run.trial199:> rpm -q kernel | sort -rV | head -n 1 2026-02-20T20:58:06.614 INFO:teuthology.orchestra.run.trial199.stdout:kernel-5.14.0-681.el9.x86_64 2026-02-20T20:58:06.615 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-681.el9.x86_64 2026-02-20T20:58:06.615 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2026-02-20T20:58:06.616 INFO:teuthology.task.kernel:Enabling kdb on cluster2.mon.a... 2026-02-20T20:58:06.616 DEBUG:teuthology.orchestra.run.trial199:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-20T20:58:06.679 INFO:teuthology.orchestra.run.trial199.stdout:ttyS1 2026-02-20T20:58:06.700 DEBUG:teuthology.parallel:result is None 2026-02-20T20:58:07.064 INFO:teuthology.orchestra.run.trial196.stdout:CentOS Stream 9 - CRB 878 kB/s | 7.8 MB 00:09 2026-02-20T20:58:08.633 INFO:teuthology.orchestra.run.trial196.stdout:CentOS Stream 9 - Extras packages 24 kB/s | 20 kB 00:00 2026-02-20T20:58:09.295 INFO:teuthology.orchestra.run.trial196.stdout:Extra Packages for Enterprise Linux 34 MB/s | 20 MB 00:00 2026-02-20T20:58:12.435 INFO:teuthology.orchestra.run.trial196.stdout:lab-extras 2.4 MB/s | 50 kB 00:00 2026-02-20T20:58:13.448 INFO:teuthology.orchestra.run.trial196.stdout:Package kernel-5.14.0-645.el9.x86_64 is already installed. 2026-02-20T20:58:13.448 INFO:teuthology.orchestra.run.trial196.stdout:Package kernel-5.14.0-681.el9.x86_64 is already installed. 2026-02-20T20:58:13.467 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T20:58:13.468 INFO:teuthology.orchestra.run.trial196.stdout:Nothing to do. 2026-02-20T20:58:13.468 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T20:58:13.492 DEBUG:teuthology.orchestra.run.trial196:> echo no | sudo yum reinstall kernel || true 2026-02-20T20:58:13.830 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:01 ago on Fri 20 Feb 2026 08:58:12 PM UTC. 2026-02-20T20:58:13.911 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T20:58:13.912 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T20:58:13.912 INFO:teuthology.orchestra.run.trial196.stdout: Package Architecture Version Repository Size 2026-02-20T20:58:13.912 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T20:58:13.912 INFO:teuthology.orchestra.run.trial196.stdout:Reinstalling: 2026-02-20T20:58:13.912 INFO:teuthology.orchestra.run.trial196.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-20T20:58:13.913 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T20:58:13.913 INFO:teuthology.orchestra.run.trial196.stdout:Transaction Summary 2026-02-20T20:58:13.913 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T20:58:13.913 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T20:58:13.913 INFO:teuthology.orchestra.run.trial196.stdout:Total download size: 803 k 2026-02-20T20:58:13.913 INFO:teuthology.orchestra.run.trial196.stdout:Installed size: 0 2026-02-20T20:58:13.913 INFO:teuthology.orchestra.run.trial196.stderr:Operation aborted. 2026-02-20T20:58:13.932 INFO:teuthology.orchestra.run.trial196.stdout:Is this ok [y/N]: 2026-02-20T20:58:13.933 DEBUG:teuthology.orchestra.run.trial196:> sudo yum reinstall -y kernel || true 2026-02-20T20:58:14.231 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:02 ago on Fri 20 Feb 2026 08:58:12 PM UTC. 2026-02-20T20:58:14.313 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T20:58:14.314 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T20:58:14.314 INFO:teuthology.orchestra.run.trial196.stdout: Package Architecture Version Repository Size 2026-02-20T20:58:14.314 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T20:58:14.314 INFO:teuthology.orchestra.run.trial196.stdout:Reinstalling: 2026-02-20T20:58:14.314 INFO:teuthology.orchestra.run.trial196.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-20T20:58:14.314 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T20:58:14.314 INFO:teuthology.orchestra.run.trial196.stdout:Transaction Summary 2026-02-20T20:58:14.315 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T20:58:14.315 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T20:58:14.315 INFO:teuthology.orchestra.run.trial196.stdout:Total download size: 803 k 2026-02-20T20:58:14.315 INFO:teuthology.orchestra.run.trial196.stdout:Installed size: 0 2026-02-20T20:58:14.315 INFO:teuthology.orchestra.run.trial196.stdout:Downloading Packages: 2026-02-20T20:58:15.770 INFO:teuthology.orchestra.run.trial196.stdout:kernel-5.14.0-681.el9.x86_64.rpm 585 kB/s | 803 kB 00:01 2026-02-20T20:58:15.770 INFO:teuthology.orchestra.run.trial196.stdout:-------------------------------------------------------------------------------- 2026-02-20T20:58:15.770 INFO:teuthology.orchestra.run.trial196.stdout:Total 551 kB/s | 803 kB 00:01 2026-02-20T20:58:15.780 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction check 2026-02-20T20:58:15.813 INFO:teuthology.orchestra.run.trial196.stdout:Transaction check succeeded. 2026-02-20T20:58:15.813 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction test 2026-02-20T20:58:15.818 INFO:teuthology.orchestra.run.trial196.stdout:Transaction test succeeded. 2026-02-20T20:58:15.819 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction 2026-02-20T20:58:15.842 INFO:teuthology.orchestra.run.trial196.stdout: Preparing : 1/1 2026-02-20T20:58:15.846 INFO:teuthology.orchestra.run.trial196.stdout: Reinstalling : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-20T20:58:15.846 INFO:teuthology.orchestra.run.trial196.stdout: Cleanup : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-20T20:58:15.953 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: kernel-5.14.0-681.el9.x86_64 2/2 2026-02-20T20:58:15.953 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-20T20:58:16.009 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-20T20:58:16.009 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T20:58:16.009 INFO:teuthology.orchestra.run.trial196.stdout:Reinstalled: 2026-02-20T20:58:16.009 INFO:teuthology.orchestra.run.trial196.stdout: kernel-5.14.0-681.el9.x86_64 2026-02-20T20:58:16.010 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T20:58:16.010 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T20:58:16.037 DEBUG:teuthology.orchestra.run.trial196:> rpm -q kernel | sort -rV | head -n 1 2026-02-20T20:58:16.061 INFO:teuthology.orchestra.run.trial196.stdout:kernel-5.14.0-681.el9.x86_64 2026-02-20T20:58:16.061 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-681.el9.x86_64 2026-02-20T20:58:16.062 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2026-02-20T20:58:16.062 INFO:teuthology.task.kernel:Enabling kdb on cluster1.mon.a... 2026-02-20T20:58:16.062 DEBUG:teuthology.orchestra.run.trial196:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-20T20:58:16.125 INFO:teuthology.orchestra.run.trial196.stdout:ttyS1 2026-02-20T20:58:16.151 DEBUG:teuthology.parallel:result is None 2026-02-20T20:58:16.152 INFO:teuthology.run_tasks:Running task internal.base... 2026-02-20T20:58:16.158 INFO:teuthology.task.internal:Creating test directory... 2026-02-20T20:58:16.159 DEBUG:teuthology.orchestra.run.trial196:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-20T20:58:16.194 DEBUG:teuthology.orchestra.run.trial199:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-20T20:58:16.210 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2026-02-20T20:58:16.222 INFO:teuthology.run_tasks:Running task internal.archive... 2026-02-20T20:58:16.235 INFO:teuthology.task.internal:Creating archive directory... 2026-02-20T20:58:16.235 DEBUG:teuthology.orchestra.run.trial196:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-20T20:58:16.250 DEBUG:teuthology.orchestra.run.trial199:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-20T20:58:16.277 INFO:teuthology.run_tasks:Running task internal.coredump... 2026-02-20T20:58:16.288 INFO:teuthology.task.internal:Enabling coredump saving... 2026-02-20T20:58:16.288 DEBUG:teuthology.orchestra.run.trial196:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-20T20:58:16.307 DEBUG:teuthology.orchestra.run.trial199:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-20T20:58:16.329 INFO:teuthology.orchestra.run.trial196.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-20T20:58:16.331 INFO:teuthology.orchestra.run.trial199.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-20T20:58:16.337 INFO:teuthology.orchestra.run.trial196.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-20T20:58:16.341 INFO:teuthology.orchestra.run.trial199.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-20T20:58:16.342 INFO:teuthology.run_tasks:Running task internal.sudo... 2026-02-20T20:58:16.354 INFO:teuthology.task.internal:Configuring sudo... 2026-02-20T20:58:16.354 DEBUG:teuthology.orchestra.run.trial196:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-20T20:58:16.382 DEBUG:teuthology.orchestra.run.trial199:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-20T20:58:16.406 INFO:teuthology.run_tasks:Running task internal.syslog... 2026-02-20T20:58:16.426 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2026-02-20T20:58:16.427 DEBUG:teuthology.orchestra.run.trial196:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-20T20:58:16.447 DEBUG:teuthology.orchestra.run.trial199:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-20T20:58:16.463 DEBUG:teuthology.orchestra.run.trial196:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-20T20:58:16.621 DEBUG:teuthology.orchestra.run.trial196:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-20T20:58:16.645 DEBUG:teuthology.orchestra.run.trial196:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-20T20:58:16.788 DEBUG:teuthology.orchestra.run.trial196:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-20T20:58:16.810 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T20:58:16.811 DEBUG:teuthology.orchestra.run.trial196:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-20T20:58:16.874 DEBUG:teuthology.orchestra.run.trial199:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-20T20:58:16.963 DEBUG:teuthology.orchestra.run.trial199:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-20T20:58:16.988 DEBUG:teuthology.orchestra.run.trial199:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-20T20:58:17.130 DEBUG:teuthology.orchestra.run.trial199:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-20T20:58:17.153 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T20:58:17.153 DEBUG:teuthology.orchestra.run.trial199:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-20T20:58:17.218 DEBUG:teuthology.orchestra.run.trial196:> sudo service rsyslog restart 2026-02-20T20:58:17.220 DEBUG:teuthology.orchestra.run.trial199:> sudo service rsyslog restart 2026-02-20T20:58:17.245 INFO:teuthology.orchestra.run.trial196.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-20T20:58:17.285 INFO:teuthology.orchestra.run.trial199.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-20T20:58:17.728 INFO:teuthology.run_tasks:Running task internal.timer... 2026-02-20T20:58:17.758 INFO:teuthology.task.internal:Starting timer... 2026-02-20T20:58:17.759 INFO:teuthology.run_tasks:Running task pcp... 2026-02-20T20:58:17.799 INFO:teuthology.run_tasks:Running task selinux... 2026-02-20T20:58:17.820 DEBUG:teuthology.orchestra.run.trial196:> sudo service auditd rotate 2026-02-20T20:58:17.851 INFO:teuthology.orchestra.run.trial196.stdout:Rotating logs: 2026-02-20T20:58:17.852 DEBUG:teuthology.orchestra.run.trial199:> sudo service auditd rotate 2026-02-20T20:58:17.881 INFO:teuthology.orchestra.run.trial199.stdout:Rotating logs: 2026-02-20T20:58:17.883 DEBUG:teuthology.task.selinux:Getting current SELinux state 2026-02-20T20:58:17.883 DEBUG:teuthology.orchestra.run.trial196:> /usr/sbin/getenforce 2026-02-20T20:58:17.909 INFO:teuthology.orchestra.run.trial196.stdout:Permissive 2026-02-20T20:58:17.909 DEBUG:teuthology.orchestra.run.trial199:> /usr/sbin/getenforce 2026-02-20T20:58:17.939 INFO:teuthology.orchestra.run.trial199.stdout:Permissive 2026-02-20T20:58:17.939 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@trial196.front.sepia.ceph.com': 'permissive', 'ubuntu@trial199.front.sepia.ceph.com': 'permissive'} 2026-02-20T20:58:17.940 DEBUG:teuthology.orchestra.run.trial196:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2026-02-20T20:58:17.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T20:58:17.974 DEBUG:teuthology.orchestra.run.trial199:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2026-02-20T20:58:18.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T20:58:18.005 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2026-02-20T20:58:18.005 DEBUG:teuthology.orchestra.run.trial196:> sudo /usr/sbin/setenforce permissive 2026-02-20T20:58:18.041 DEBUG:teuthology.orchestra.run.trial199:> sudo /usr/sbin/setenforce permissive 2026-02-20T20:58:18.068 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2026-02-20T20:58:18.083 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2026-02-20T20:58:18.086 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2026-02-20T20:58:18.190 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2026-02-20T20:58:18.203 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2026-02-20T20:58:18.204 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit trial196.front.sepia.ceph.com,trial199.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2026-02-20T21:01:01.066 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@trial196.front.sepia.ceph.com'), Remote(name='ubuntu@trial199.front.sepia.ceph.com')] 2026-02-20T21:01:01.067 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial196.front.sepia.ceph.com' 2026-02-20T21:01:01.069 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial196.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-20T21:01:01.171 DEBUG:teuthology.orchestra.run.trial196:> true 2026-02-20T21:01:01.246 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial196.front.sepia.ceph.com' 2026-02-20T21:01:01.246 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial199.front.sepia.ceph.com' 2026-02-20T21:01:01.247 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial199.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-20T21:01:01.334 DEBUG:teuthology.orchestra.run.trial199:> true 2026-02-20T21:01:01.405 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial199.front.sepia.ceph.com' 2026-02-20T21:01:01.406 INFO:teuthology.run_tasks:Running task clock... 2026-02-20T21:01:01.422 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2026-02-20T21:01:01.422 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-20T21:01:01.422 DEBUG:teuthology.orchestra.run.trial196:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-20T21:01:01.424 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-20T21:01:01.424 DEBUG:teuthology.orchestra.run.trial199:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-20T21:01:01.450 INFO:teuthology.orchestra.run.trial196.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2026-02-20T21:01:01.462 INFO:teuthology.orchestra.run.trial196.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2026-02-20T21:01:01.472 INFO:teuthology.orchestra.run.trial199.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2026-02-20T21:01:01.484 INFO:teuthology.orchestra.run.trial199.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2026-02-20T21:01:01.484 INFO:teuthology.orchestra.run.trial196.stderr:sudo: ntpd: command not found 2026-02-20T21:01:01.493 INFO:teuthology.orchestra.run.trial196.stdout:506 Cannot talk to daemon 2026-02-20T21:01:01.504 INFO:teuthology.orchestra.run.trial199.stderr:sudo: ntpd: command not found 2026-02-20T21:01:01.505 INFO:teuthology.orchestra.run.trial196.stderr:Failed to start ntp.service: Unit ntp.service not found. 2026-02-20T21:01:01.513 INFO:teuthology.orchestra.run.trial199.stdout:506 Cannot talk to daemon 2026-02-20T21:01:01.516 INFO:teuthology.orchestra.run.trial196.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2026-02-20T21:01:01.524 INFO:teuthology.orchestra.run.trial199.stderr:Failed to start ntp.service: Unit ntp.service not found. 2026-02-20T21:01:01.536 INFO:teuthology.orchestra.run.trial199.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2026-02-20T21:01:01.561 INFO:teuthology.orchestra.run.trial196.stderr:bash: line 1: ntpq: command not found 2026-02-20T21:01:01.563 INFO:teuthology.orchestra.run.trial196.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-20T21:01:01.563 INFO:teuthology.orchestra.run.trial196.stdout:=============================================================================== 2026-02-20T21:01:01.583 INFO:teuthology.orchestra.run.trial199.stderr:bash: line 1: ntpq: command not found 2026-02-20T21:01:01.603 INFO:teuthology.orchestra.run.trial199.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-20T21:01:01.603 INFO:teuthology.orchestra.run.trial199.stdout:=============================================================================== 2026-02-20T21:01:01.603 INFO:teuthology.orchestra.run.trial199.stdout:^? 139-177-202-26.ip.linode> 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-20T21:01:01.603 INFO:teuthology.orchestra.run.trial199.stdout:^? 45-79-189-79.ip.linodeus> 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-20T21:01:01.604 INFO:teuthology.orchestra.run.trial199.stdout:^? s2-a.time.mci1.us.rozint> 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-20T21:01:01.604 INFO:teuthology.orchestra.run.trial199.stdout:^? 69-10-208-170.rainiercon> 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-20T21:01:01.605 INFO:teuthology.run_tasks:Running task install... 2026-02-20T21:01:01.621 DEBUG:teuthology.task.install:project ceph 2026-02-20T21:01:01.621 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}} 2026-02-20T21:01:01.621 DEBUG:teuthology.task.install:config {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'} 2026-02-20T21:01:01.621 INFO:teuthology.task.install:Using flavor: default 2026-02-20T21:01:01.630 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2026-02-20T21:01:01.631 INFO:teuthology.task.install:extra packages: ['rbd-mirror'] 2026-02-20T21:01:01.631 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'f7870454149f22574086dd2ff55d050585136531', 'tag': None, 'wait_for_package': False} 2026-02-20T21:01:01.632 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f7870454149f22574086dd2ff55d050585136531 2026-02-20T21:01:01.634 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'f7870454149f22574086dd2ff55d050585136531', 'tag': None, 'wait_for_package': False} 2026-02-20T21:01:01.635 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f7870454149f22574086dd2ff55d050585136531 2026-02-20T21:01:01.833 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/centos/9/flavors/default/ 2026-02-20T21:01:01.833 INFO:teuthology.task.install.rpm:Package version is 20.2.0-677.gf7870454 2026-02-20T21:01:01.845 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/centos/9/flavors/default/ 2026-02-20T21:01:01.845 INFO:teuthology.task.install.rpm:Package version is 20.2.0-677.gf7870454 2026-02-20T21:01:01.992 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2026-02-20T21:01:01.992 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T21:01:01.992 DEBUG:teuthology.orchestra.run.trial196:> sudo dd of=/etc/yum.repos.d/ceph.repo 2026-02-20T21:01:02.003 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2026-02-20T21:01:02.003 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:01:02.003 DEBUG:teuthology.orchestra.run.trial199:> sudo dd of=/etc/yum.repos.d/ceph.repo 2026-02-20T21:01:02.017 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on remote rpm x86_64 2026-02-20T21:01:02.017 DEBUG:teuthology.orchestra.run.trial196:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/f7870454149f22574086dd2ff55d050585136531/;g' /etc/yum.repos.d/ceph.repo ; fi 2026-02-20T21:01:02.028 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on remote rpm x86_64 2026-02-20T21:01:02.028 DEBUG:teuthology.orchestra.run.trial199:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/f7870454149f22574086dd2ff55d050585136531/;g' /etc/yum.repos.d/ceph.repo ; fi 2026-02-20T21:01:02.084 DEBUG:teuthology.orchestra.run.trial196:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2026-02-20T21:01:02.093 DEBUG:teuthology.orchestra.run.trial199:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2026-02-20T21:01:02.159 DEBUG:teuthology.orchestra.run.trial196:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2026-02-20T21:01:02.167 DEBUG:teuthology.orchestra.run.trial199:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2026-02-20T21:01:02.222 INFO:teuthology.orchestra.run.trial196.stdout:check_obsoletes = 1 2026-02-20T21:01:02.224 DEBUG:teuthology.orchestra.run.trial196:> sudo yum clean all 2026-02-20T21:01:02.230 INFO:teuthology.orchestra.run.trial199.stdout:check_obsoletes = 1 2026-02-20T21:01:02.232 DEBUG:teuthology.orchestra.run.trial199:> sudo yum clean all 2026-02-20T21:01:02.382 INFO:teuthology.orchestra.run.trial196.stdout:41 files removed 2026-02-20T21:01:02.392 INFO:teuthology.orchestra.run.trial199.stdout:41 files removed 2026-02-20T21:01:02.397 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install ceph-radosgw 2026-02-20T21:01:02.407 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install ceph-radosgw 2026-02-20T21:01:02.667 INFO:teuthology.orchestra.run.trial196.stdout:ceph packages for x86_64 678 kB/s | 87 kB 00:00 2026-02-20T21:01:02.681 INFO:teuthology.orchestra.run.trial199.stdout:ceph packages for x86_64 671 kB/s | 87 kB 00:00 2026-02-20T21:01:02.809 INFO:teuthology.orchestra.run.trial196.stdout:ceph noarch packages 156 kB/s | 18 kB 00:00 2026-02-20T21:01:02.824 INFO:teuthology.orchestra.run.trial199.stdout:ceph noarch packages 155 kB/s | 18 kB 00:00 2026-02-20T21:01:02.936 INFO:teuthology.orchestra.run.trial196.stdout:ceph source packages 18 kB/s | 1.9 kB 00:00 2026-02-20T21:01:02.953 INFO:teuthology.orchestra.run.trial199.stdout:ceph source packages 18 kB/s | 1.9 kB 00:00 2026-02-20T21:01:03.637 INFO:teuthology.orchestra.run.trial199.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.9 MB 00:00 2026-02-20T21:01:06.317 INFO:teuthology.orchestra.run.trial199.stdout:CentOS Stream 9 - AppStream 12 MB/s | 27 MB 00:02 2026-02-20T21:01:10.183 INFO:teuthology.orchestra.run.trial199.stdout:CentOS Stream 9 - CRB 5.1 MB/s | 7.8 MB 00:01 2026-02-20T21:01:11.206 INFO:teuthology.orchestra.run.trial199.stdout:CentOS Stream 9 - Extras packages 66 kB/s | 20 kB 00:00 2026-02-20T21:01:11.704 INFO:teuthology.orchestra.run.trial196.stdout:CentOS Stream 9 - BaseOS 1.0 MB/s | 8.9 MB 00:08 2026-02-20T21:01:11.922 INFO:teuthology.orchestra.run.trial199.stdout:Extra Packages for Enterprise Linux 31 MB/s | 20 MB 00:00 2026-02-20T21:01:12.905 INFO:teuthology.orchestra.run.trial196.stdout:CentOS Stream 9 - AppStream 41 MB/s | 27 MB 00:00 2026-02-20T21:01:15.095 INFO:teuthology.orchestra.run.trial199.stdout:lab-extras 1.7 MB/s | 50 kB 00:00 2026-02-20T21:01:16.285 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:01:16.286 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:16.287 INFO:teuthology.orchestra.run.trial199.stdout: Package Arch Version Repository Size 2026-02-20T21:01:16.287 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:16.287 INFO:teuthology.orchestra.run.trial199.stdout:Installing: 2026-02-20T21:01:16.287 INFO:teuthology.orchestra.run.trial199.stdout: ceph-radosgw x86_64 2:20.2.0-677.gf7870454.el9 ceph 24 M 2026-02-20T21:01:16.287 INFO:teuthology.orchestra.run.trial199.stdout:Upgrading: 2026-02-20T21:01:16.287 INFO:teuthology.orchestra.run.trial199.stdout: librados2 x86_64 2:20.2.0-677.gf7870454.el9 ceph 3.5 M 2026-02-20T21:01:16.287 INFO:teuthology.orchestra.run.trial199.stdout: librbd1 x86_64 2:20.2.0-677.gf7870454.el9 ceph 2.8 M 2026-02-20T21:01:16.287 INFO:teuthology.orchestra.run.trial199.stdout:Installing dependencies: 2026-02-20T21:01:16.288 INFO:teuthology.orchestra.run.trial199.stdout: boost-program-options x86_64 1.75.0-13.el9 appstream 104 k 2026-02-20T21:01:16.288 INFO:teuthology.orchestra.run.trial199.stdout: ceph-base x86_64 2:20.2.0-677.gf7870454.el9 ceph 5.9 M 2026-02-20T21:01:16.288 INFO:teuthology.orchestra.run.trial199.stdout: ceph-common x86_64 2:20.2.0-677.gf7870454.el9 ceph 24 M 2026-02-20T21:01:16.288 INFO:teuthology.orchestra.run.trial199.stdout: ceph-selinux x86_64 2:20.2.0-677.gf7870454.el9 ceph 25 k 2026-02-20T21:01:16.288 INFO:teuthology.orchestra.run.trial199.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2026-02-20T21:01:16.288 INFO:teuthology.orchestra.run.trial199.stdout: libarrow x86_64 9.0.0-15.el9 epel 4.4 M 2026-02-20T21:01:16.288 INFO:teuthology.orchestra.run.trial199.stdout: libarrow-doc noarch 9.0.0-15.el9 epel 25 k 2026-02-20T21:01:16.288 INFO:teuthology.orchestra.run.trial199.stdout: libcephfs2 x86_64 2:20.2.0-677.gf7870454.el9 ceph 869 k 2026-02-20T21:01:16.288 INFO:teuthology.orchestra.run.trial199.stdout: libnbd x86_64 1.20.3-4.el9 appstream 164 k 2026-02-20T21:01:16.288 INFO:teuthology.orchestra.run.trial199.stdout: liboath x86_64 2.6.12-1.el9 epel 49 k 2026-02-20T21:01:16.288 INFO:teuthology.orchestra.run.trial199.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2026-02-20T21:01:16.288 INFO:teuthology.orchestra.run.trial199.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2026-02-20T21:01:16.289 INFO:teuthology.orchestra.run.trial199.stdout: libradosstriper1 x86_64 2:20.2.0-677.gf7870454.el9 ceph 250 k 2026-02-20T21:01:16.289 INFO:teuthology.orchestra.run.trial199.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2026-02-20T21:01:16.289 INFO:teuthology.orchestra.run.trial199.stdout: librgw2 x86_64 2:20.2.0-677.gf7870454.el9 ceph 6.4 M 2026-02-20T21:01:16.289 INFO:teuthology.orchestra.run.trial199.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2026-02-20T21:01:16.289 INFO:teuthology.orchestra.run.trial199.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2026-02-20T21:01:16.289 INFO:teuthology.orchestra.run.trial199.stdout: mailcap noarch 2.1.49-5.el9 baseos 33 k 2026-02-20T21:01:16.289 INFO:teuthology.orchestra.run.trial199.stdout: parquet-libs x86_64 9.0.0-15.el9 epel 838 k 2026-02-20T21:01:16.289 INFO:teuthology.orchestra.run.trial199.stdout: pciutils x86_64 3.7.0-7.el9 baseos 93 k 2026-02-20T21:01:16.289 INFO:teuthology.orchestra.run.trial199.stdout: python3-ceph-argparse x86_64 2:20.2.0-677.gf7870454.el9 ceph 45 k 2026-02-20T21:01:16.289 INFO:teuthology.orchestra.run.trial199.stdout: python3-ceph-common x86_64 2:20.2.0-677.gf7870454.el9 ceph 175 k 2026-02-20T21:01:16.289 INFO:teuthology.orchestra.run.trial199.stdout: python3-cephfs x86_64 2:20.2.0-677.gf7870454.el9 ceph 163 k 2026-02-20T21:01:16.289 INFO:teuthology.orchestra.run.trial199.stdout: python3-rados x86_64 2:20.2.0-677.gf7870454.el9 ceph 323 k 2026-02-20T21:01:16.289 INFO:teuthology.orchestra.run.trial199.stdout: python3-rbd x86_64 2:20.2.0-677.gf7870454.el9 ceph 305 k 2026-02-20T21:01:16.290 INFO:teuthology.orchestra.run.trial199.stdout: python3-rgw x86_64 2:20.2.0-677.gf7870454.el9 ceph 99 k 2026-02-20T21:01:16.290 INFO:teuthology.orchestra.run.trial199.stdout: qatlib x86_64 25.08.0-2.el9 appstream 240 k 2026-02-20T21:01:16.290 INFO:teuthology.orchestra.run.trial199.stdout: qatzip-libs x86_64 1.3.1-1.el9 appstream 66 k 2026-02-20T21:01:16.290 INFO:teuthology.orchestra.run.trial199.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2026-02-20T21:01:16.290 INFO:teuthology.orchestra.run.trial199.stdout: thrift x86_64 0.15.0-4.el9 epel 1.6 M 2026-02-20T21:01:16.290 INFO:teuthology.orchestra.run.trial199.stdout:Installing weak dependencies: 2026-02-20T21:01:16.290 INFO:teuthology.orchestra.run.trial199.stdout: qatlib-service x86_64 25.08.0-2.el9 appstream 37 k 2026-02-20T21:01:16.290 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:16.290 INFO:teuthology.orchestra.run.trial199.stdout:Transaction Summary 2026-02-20T21:01:16.290 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:16.290 INFO:teuthology.orchestra.run.trial199.stdout:Install 32 Packages 2026-02-20T21:01:16.290 INFO:teuthology.orchestra.run.trial199.stdout:Upgrade 2 Packages 2026-02-20T21:01:16.291 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:16.291 INFO:teuthology.orchestra.run.trial199.stdout:Total download size: 78 M 2026-02-20T21:01:16.291 INFO:teuthology.orchestra.run.trial199.stdout:Downloading Packages: 2026-02-20T21:01:16.465 INFO:teuthology.orchestra.run.trial196.stdout:CentOS Stream 9 - CRB 6.5 MB/s | 7.8 MB 00:01 2026-02-20T21:01:16.918 INFO:teuthology.orchestra.run.trial199.stdout:(1/34): ceph-base-20.2.0-677.gf7870454.el9.x86_ 14 MB/s | 5.9 MB 00:00 2026-02-20T21:01:16.955 INFO:teuthology.orchestra.run.trial199.stdout:(2/34): ceph-selinux-20.2.0-677.gf7870454.el9.x 679 kB/s | 25 kB 00:00 2026-02-20T21:01:17.010 INFO:teuthology.orchestra.run.trial199.stdout:(3/34): libcephfs2-20.2.0-677.gf7870454.el9.x86 15 MB/s | 869 kB 00:00 2026-02-20T21:01:17.054 INFO:teuthology.orchestra.run.trial199.stdout:(4/34): libradosstriper1-20.2.0-677.gf7870454.e 5.7 MB/s | 250 kB 00:00 2026-02-20T21:01:17.442 INFO:teuthology.orchestra.run.trial199.stdout:(5/34): librgw2-20.2.0-677.gf7870454.el9.x86_64 16 MB/s | 6.4 MB 00:00 2026-02-20T21:01:17.483 INFO:teuthology.orchestra.run.trial199.stdout:(6/34): python3-ceph-argparse-20.2.0-677.gf7870 1.1 MB/s | 45 kB 00:00 2026-02-20T21:01:17.527 INFO:teuthology.orchestra.run.trial199.stdout:(7/34): python3-ceph-common-20.2.0-677.gf787045 3.9 MB/s | 175 kB 00:00 2026-02-20T21:01:17.570 INFO:teuthology.orchestra.run.trial199.stdout:(8/34): python3-cephfs-20.2.0-677.gf7870454.el9 3.7 MB/s | 163 kB 00:00 2026-02-20T21:01:17.616 INFO:teuthology.orchestra.run.trial199.stdout:(9/34): ceph-common-20.2.0-677.gf7870454.el9.x8 21 MB/s | 24 MB 00:01 2026-02-20T21:01:17.617 INFO:teuthology.orchestra.run.trial199.stdout:(10/34): python3-rados-20.2.0-677.gf7870454.el9 6.6 MB/s | 323 kB 00:00 2026-02-20T21:01:17.667 INFO:teuthology.orchestra.run.trial199.stdout:(11/34): ceph-radosgw-20.2.0-677.gf7870454.el9. 20 MB/s | 24 MB 00:01 2026-02-20T21:01:17.668 INFO:teuthology.orchestra.run.trial199.stdout:(12/34): python3-rgw-20.2.0-677.gf7870454.el9.x 1.9 MB/s | 99 kB 00:00 2026-02-20T21:01:17.669 INFO:teuthology.orchestra.run.trial199.stdout:(13/34): python3-rbd-20.2.0-677.gf7870454.el9.x 5.6 MB/s | 305 kB 00:00 2026-02-20T21:01:17.775 INFO:teuthology.orchestra.run.trial199.stdout:(14/34): boost-program-options-1.75.0-13.el9.x8 978 kB/s | 104 kB 00:00 2026-02-20T21:01:17.817 INFO:teuthology.orchestra.run.trial199.stdout:(15/34): libnbd-1.20.3-4.el9.x86_64.rpm 3.8 MB/s | 164 kB 00:00 2026-02-20T21:01:17.928 INFO:teuthology.orchestra.run.trial199.stdout:(16/34): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.4 MB/s | 160 kB 00:00 2026-02-20T21:01:17.950 INFO:teuthology.orchestra.run.trial199.stdout:(17/34): librabbitmq-0.11.0-7.el9.x86_64.rpm 2.1 MB/s | 45 kB 00:00 2026-02-20T21:01:18.000 INFO:teuthology.orchestra.run.trial199.stdout:(18/34): librdkafka-1.6.1-102.el9.x86_64.rpm 13 MB/s | 662 kB 00:00 2026-02-20T21:01:18.024 INFO:teuthology.orchestra.run.trial199.stdout:(19/34): lttng-ust-2.12.0-6.el9.x86_64.rpm 12 MB/s | 292 kB 00:00 2026-02-20T21:01:18.048 INFO:teuthology.orchestra.run.trial199.stdout:(20/34): qatlib-25.08.0-2.el9.x86_64.rpm 10 MB/s | 240 kB 00:00 2026-02-20T21:01:18.073 INFO:teuthology.orchestra.run.trial199.stdout:(21/34): qatlib-service-25.08.0-2.el9.x86_64.rp 1.4 MB/s | 37 kB 00:00 2026-02-20T21:01:18.090 INFO:teuthology.orchestra.run.trial196.stdout:CentOS Stream 9 - Extras packages 22 kB/s | 20 kB 00:00 2026-02-20T21:01:18.110 INFO:teuthology.orchestra.run.trial199.stdout:(22/34): qatzip-libs-1.3.1-1.el9.x86_64.rpm 1.7 MB/s | 66 kB 00:00 2026-02-20T21:01:18.111 INFO:teuthology.orchestra.run.trial199.stdout:(23/34): mailcap-2.1.49-5.el9.noarch.rpm 75 kB/s | 33 kB 00:00 2026-02-20T21:01:18.184 INFO:teuthology.orchestra.run.trial199.stdout:(24/34): gperftools-libs-2.9.1-3.el9.x86_64.rpm 4.1 MB/s | 308 kB 00:00 2026-02-20T21:01:18.193 INFO:teuthology.orchestra.run.trial199.stdout:(25/34): libarrow-doc-9.0.0-15.el9.noarch.rpm 2.6 MB/s | 25 kB 00:00 2026-02-20T21:01:18.203 INFO:teuthology.orchestra.run.trial199.stdout:(26/34): liboath-2.6.12-1.el9.x86_64.rpm 4.9 MB/s | 49 kB 00:00 2026-02-20T21:01:18.211 INFO:teuthology.orchestra.run.trial199.stdout:(27/34): libarrow-9.0.0-15.el9.x86_64.rpm 44 MB/s | 4.4 MB 00:00 2026-02-20T21:01:18.213 INFO:teuthology.orchestra.run.trial199.stdout:(28/34): libunwind-1.6.2-1.el9.x86_64.rpm 6.5 MB/s | 67 kB 00:00 2026-02-20T21:01:18.235 INFO:teuthology.orchestra.run.trial199.stdout:(29/34): parquet-libs-9.0.0-15.el9.x86_64.rpm 33 MB/s | 838 kB 00:00 2026-02-20T21:01:18.237 INFO:teuthology.orchestra.run.trial199.stdout:(30/34): re2-20211101-20.el9.x86_64.rpm 7.9 MB/s | 191 kB 00:00 2026-02-20T21:01:18.256 INFO:teuthology.orchestra.run.trial199.stdout:(31/34): thrift-0.15.0-4.el9.x86_64.rpm 77 MB/s | 1.6 MB 00:00 2026-02-20T21:01:18.262 INFO:teuthology.orchestra.run.trial199.stdout:(32/34): pciutils-3.7.0-7.el9.x86_64.rpm 157 kB/s | 93 kB 00:00 2026-02-20T21:01:18.357 INFO:teuthology.orchestra.run.trial199.stdout:(33/34): librbd1-20.2.0-677.gf7870454.el9.x86_6 28 MB/s | 2.8 MB 00:00 2026-02-20T21:01:18.361 INFO:teuthology.orchestra.run.trial199.stdout:(34/34): librados2-20.2.0-677.gf7870454.el9.x86 28 MB/s | 3.5 MB 00:00 2026-02-20T21:01:18.364 INFO:teuthology.orchestra.run.trial199.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:01:18.364 INFO:teuthology.orchestra.run.trial199.stdout:Total 37 MB/s | 78 MB 00:02 2026-02-20T21:01:18.450 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction check 2026-02-20T21:01:18.467 INFO:teuthology.orchestra.run.trial199.stdout:Transaction check succeeded. 2026-02-20T21:01:18.467 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction test 2026-02-20T21:01:18.763 INFO:teuthology.orchestra.run.trial199.stdout:Transaction test succeeded. 2026-02-20T21:01:18.763 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction 2026-02-20T21:01:19.104 INFO:teuthology.orchestra.run.trial199.stdout: Preparing : 1/1 2026-02-20T21:01:19.117 INFO:teuthology.orchestra.run.trial199.stdout: Installing : thrift-0.15.0-4.el9.x86_64 1/36 2026-02-20T21:01:19.272 INFO:teuthology.orchestra.run.trial199.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 2/36 2026-02-20T21:01:19.277 INFO:teuthology.orchestra.run.trial199.stdout: Upgrading : librados2-2:20.2.0-677.gf7870454.el9.x86_64 3/36 2026-02-20T21:01:19.307 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: librados2-2:20.2.0-677.gf7870454.el9.x86_64 3/36 2026-02-20T21:01:19.345 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-rados-2:20.2.0-677.gf7870454.el9.x86_64 4/36 2026-02-20T21:01:19.346 INFO:teuthology.orchestra.run.trial199.stdout: Installing : libcephfs2-2:20.2.0-677.gf7870454.el9.x86_64 5/36 2026-02-20T21:01:19.358 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: libcephfs2-2:20.2.0-677.gf7870454.el9.x86_64 5/36 2026-02-20T21:01:19.367 INFO:teuthology.orchestra.run.trial199.stdout: Installing : liboath-2.6.12-1.el9.x86_64 6/36 2026-02-20T21:01:19.372 INFO:teuthology.orchestra.run.trial199.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 7/36 2026-02-20T21:01:19.375 INFO:teuthology.orchestra.run.trial199.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 8/36 2026-02-20T21:01:19.380 INFO:teuthology.orchestra.run.trial199.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 9/36 2026-02-20T21:01:19.388 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-ceph-argparse-2:20.2.0-677.gf7870454.el9.x 10/36 2026-02-20T21:01:19.400 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-cephfs-2:20.2.0-677.gf7870454.el9.x86_64 11/36 2026-02-20T21:01:19.401 INFO:teuthology.orchestra.run.trial199.stdout: Installing : libradosstriper1-2:20.2.0-677.gf7870454.el9.x86_64 12/36 2026-02-20T21:01:19.415 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: libradosstriper1-2:20.2.0-677.gf7870454.el9.x86_64 12/36 2026-02-20T21:01:19.450 INFO:teuthology.orchestra.run.trial199.stdout: Installing : re2-1:20211101-20.el9.x86_64 13/36 2026-02-20T21:01:19.453 INFO:teuthology.orchestra.run.trial199.stdout: Installing : libarrow-9.0.0-15.el9.x86_64 14/36 2026-02-20T21:01:19.461 INFO:teuthology.orchestra.run.trial199.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 15/36 2026-02-20T21:01:19.463 INFO:teuthology.orchestra.run.trial199.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 16/36 2026-02-20T21:01:19.468 INFO:teuthology.orchestra.run.trial199.stdout: Installing : libarrow-doc-9.0.0-15.el9.noarch 17/36 2026-02-20T21:01:19.592 INFO:teuthology.orchestra.run.trial199.stdout: Installing : libnbd-1.20.3-4.el9.x86_64 18/36 2026-02-20T21:01:19.593 INFO:teuthology.orchestra.run.trial199.stdout: Upgrading : librbd1-2:20.2.0-677.gf7870454.el9.x86_64 19/36 2026-02-20T21:01:19.619 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: librbd1-2:20.2.0-677.gf7870454.el9.x86_64 19/36 2026-02-20T21:01:19.621 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-rbd-2:20.2.0-677.gf7870454.el9.x86_64 20/36 2026-02-20T21:01:19.627 INFO:teuthology.orchestra.run.trial199.stdout: Installing : boost-program-options-1.75.0-13.el9.x86_64 21/36 2026-02-20T21:01:19.907 INFO:teuthology.orchestra.run.trial199.stdout: Installing : parquet-libs-9.0.0-15.el9.x86_64 22/36 2026-02-20T21:01:19.909 INFO:teuthology.orchestra.run.trial199.stdout: Installing : librgw2-2:20.2.0-677.gf7870454.el9.x86_64 23/36 2026-02-20T21:01:19.924 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: librgw2-2:20.2.0-677.gf7870454.el9.x86_64 23/36 2026-02-20T21:01:19.928 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-rgw-2:20.2.0-677.gf7870454.el9.x86_64 24/36 2026-02-20T21:01:19.933 INFO:teuthology.orchestra.run.trial199.stdout: Installing : pciutils-3.7.0-7.el9.x86_64 25/36 2026-02-20T21:01:19.936 INFO:teuthology.orchestra.run.trial199.stdout: Installing : qatlib-25.08.0-2.el9.x86_64 26/36 2026-02-20T21:01:19.937 INFO:teuthology.orchestra.run.trial199.stdout: Installing : qatlib-service-25.08.0-2.el9.x86_64 27/36 2026-02-20T21:01:19.955 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 27/36 2026-02-20T21:01:19.958 INFO:teuthology.orchestra.run.trial199.stdout: Installing : qatzip-libs-1.3.1-1.el9.x86_64 28/36 2026-02-20T21:01:19.993 INFO:teuthology.orchestra.run.trial199.stdout: Installing : mailcap-2.1.49-5.el9.noarch 29/36 2026-02-20T21:01:19.994 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-ceph-common-2:20.2.0-677.gf7870454.el9.x86 30/36 2026-02-20T21:01:20.321 INFO:teuthology.orchestra.run.trial196.stdout:Extra Packages for Enterprise Linux 9.4 MB/s | 20 MB 00:02 2026-02-20T21:01:21.072 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 31/36 2026-02-20T21:01:21.074 INFO:teuthology.orchestra.run.trial199.stdout: Installing : ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 31/36 2026-02-20T21:01:21.369 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 31/36 2026-02-20T21:01:21.373 INFO:teuthology.orchestra.run.trial199.stdout: Installing : ceph-base-2:20.2.0-677.gf7870454.el9.x86_64 32/36 2026-02-20T21:01:21.416 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: ceph-base-2:20.2.0-677.gf7870454.el9.x86_64 32/36 2026-02-20T21:01:21.416 INFO:teuthology.orchestra.run.trial199.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2026-02-20T21:01:21.416 INFO:teuthology.orchestra.run.trial199.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2026-02-20T21:01:21.416 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:21.420 INFO:teuthology.orchestra.run.trial199.stdout: Installing : ceph-selinux-2:20.2.0-677.gf7870454.el9.x86_64 33/36 2026-02-20T21:01:23.558 INFO:teuthology.orchestra.run.trial196.stdout:lab-extras 2.6 MB/s | 50 kB 00:00 2026-02-20T21:01:24.758 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:01:24.759 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:01:24.760 INFO:teuthology.orchestra.run.trial196.stdout: Package Arch Version Repository Size 2026-02-20T21:01:24.760 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:01:24.760 INFO:teuthology.orchestra.run.trial196.stdout:Installing: 2026-02-20T21:01:24.760 INFO:teuthology.orchestra.run.trial196.stdout: ceph-radosgw x86_64 2:20.2.0-677.gf7870454.el9 ceph 24 M 2026-02-20T21:01:24.760 INFO:teuthology.orchestra.run.trial196.stdout:Upgrading: 2026-02-20T21:01:24.760 INFO:teuthology.orchestra.run.trial196.stdout: librados2 x86_64 2:20.2.0-677.gf7870454.el9 ceph 3.5 M 2026-02-20T21:01:24.760 INFO:teuthology.orchestra.run.trial196.stdout: librbd1 x86_64 2:20.2.0-677.gf7870454.el9 ceph 2.8 M 2026-02-20T21:01:24.760 INFO:teuthology.orchestra.run.trial196.stdout:Installing dependencies: 2026-02-20T21:01:24.760 INFO:teuthology.orchestra.run.trial196.stdout: boost-program-options x86_64 1.75.0-13.el9 appstream 104 k 2026-02-20T21:01:24.761 INFO:teuthology.orchestra.run.trial196.stdout: ceph-base x86_64 2:20.2.0-677.gf7870454.el9 ceph 5.9 M 2026-02-20T21:01:24.761 INFO:teuthology.orchestra.run.trial196.stdout: ceph-common x86_64 2:20.2.0-677.gf7870454.el9 ceph 24 M 2026-02-20T21:01:24.761 INFO:teuthology.orchestra.run.trial196.stdout: ceph-selinux x86_64 2:20.2.0-677.gf7870454.el9 ceph 25 k 2026-02-20T21:01:24.761 INFO:teuthology.orchestra.run.trial196.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2026-02-20T21:01:24.761 INFO:teuthology.orchestra.run.trial196.stdout: libarrow x86_64 9.0.0-15.el9 epel 4.4 M 2026-02-20T21:01:24.761 INFO:teuthology.orchestra.run.trial196.stdout: libarrow-doc noarch 9.0.0-15.el9 epel 25 k 2026-02-20T21:01:24.761 INFO:teuthology.orchestra.run.trial196.stdout: libcephfs2 x86_64 2:20.2.0-677.gf7870454.el9 ceph 869 k 2026-02-20T21:01:24.761 INFO:teuthology.orchestra.run.trial196.stdout: libnbd x86_64 1.20.3-4.el9 appstream 164 k 2026-02-20T21:01:24.761 INFO:teuthology.orchestra.run.trial196.stdout: liboath x86_64 2.6.12-1.el9 epel 49 k 2026-02-20T21:01:24.761 INFO:teuthology.orchestra.run.trial196.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2026-02-20T21:01:24.761 INFO:teuthology.orchestra.run.trial196.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2026-02-20T21:01:24.761 INFO:teuthology.orchestra.run.trial196.stdout: libradosstriper1 x86_64 2:20.2.0-677.gf7870454.el9 ceph 250 k 2026-02-20T21:01:24.761 INFO:teuthology.orchestra.run.trial196.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2026-02-20T21:01:24.762 INFO:teuthology.orchestra.run.trial196.stdout: librgw2 x86_64 2:20.2.0-677.gf7870454.el9 ceph 6.4 M 2026-02-20T21:01:24.762 INFO:teuthology.orchestra.run.trial196.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2026-02-20T21:01:24.762 INFO:teuthology.orchestra.run.trial196.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2026-02-20T21:01:24.762 INFO:teuthology.orchestra.run.trial196.stdout: mailcap noarch 2.1.49-5.el9 baseos 33 k 2026-02-20T21:01:24.762 INFO:teuthology.orchestra.run.trial196.stdout: parquet-libs x86_64 9.0.0-15.el9 epel 838 k 2026-02-20T21:01:24.762 INFO:teuthology.orchestra.run.trial196.stdout: pciutils x86_64 3.7.0-7.el9 baseos 93 k 2026-02-20T21:01:24.762 INFO:teuthology.orchestra.run.trial196.stdout: python3-ceph-argparse x86_64 2:20.2.0-677.gf7870454.el9 ceph 45 k 2026-02-20T21:01:24.762 INFO:teuthology.orchestra.run.trial196.stdout: python3-ceph-common x86_64 2:20.2.0-677.gf7870454.el9 ceph 175 k 2026-02-20T21:01:24.762 INFO:teuthology.orchestra.run.trial196.stdout: python3-cephfs x86_64 2:20.2.0-677.gf7870454.el9 ceph 163 k 2026-02-20T21:01:24.762 INFO:teuthology.orchestra.run.trial196.stdout: python3-rados x86_64 2:20.2.0-677.gf7870454.el9 ceph 323 k 2026-02-20T21:01:24.762 INFO:teuthology.orchestra.run.trial196.stdout: python3-rbd x86_64 2:20.2.0-677.gf7870454.el9 ceph 305 k 2026-02-20T21:01:24.762 INFO:teuthology.orchestra.run.trial196.stdout: python3-rgw x86_64 2:20.2.0-677.gf7870454.el9 ceph 99 k 2026-02-20T21:01:24.763 INFO:teuthology.orchestra.run.trial196.stdout: qatlib x86_64 25.08.0-2.el9 appstream 240 k 2026-02-20T21:01:24.763 INFO:teuthology.orchestra.run.trial196.stdout: qatzip-libs x86_64 1.3.1-1.el9 appstream 66 k 2026-02-20T21:01:24.763 INFO:teuthology.orchestra.run.trial196.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2026-02-20T21:01:24.763 INFO:teuthology.orchestra.run.trial196.stdout: thrift x86_64 0.15.0-4.el9 epel 1.6 M 2026-02-20T21:01:24.763 INFO:teuthology.orchestra.run.trial196.stdout:Installing weak dependencies: 2026-02-20T21:01:24.763 INFO:teuthology.orchestra.run.trial196.stdout: qatlib-service x86_64 25.08.0-2.el9 appstream 37 k 2026-02-20T21:01:24.763 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:24.763 INFO:teuthology.orchestra.run.trial196.stdout:Transaction Summary 2026-02-20T21:01:24.763 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:01:24.763 INFO:teuthology.orchestra.run.trial196.stdout:Install 32 Packages 2026-02-20T21:01:24.763 INFO:teuthology.orchestra.run.trial196.stdout:Upgrade 2 Packages 2026-02-20T21:01:24.764 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:24.764 INFO:teuthology.orchestra.run.trial196.stdout:Total download size: 78 M 2026-02-20T21:01:24.764 INFO:teuthology.orchestra.run.trial196.stdout:Downloading Packages: 2026-02-20T21:01:25.652 INFO:teuthology.orchestra.run.trial196.stdout:(1/34): ceph-base-20.2.0-677.gf7870454.el9.x86_ 16 MB/s | 5.9 MB 00:00 2026-02-20T21:01:25.699 INFO:teuthology.orchestra.run.trial196.stdout:(2/34): ceph-selinux-20.2.0-677.gf7870454.el9.x 546 kB/s | 25 kB 00:00 2026-02-20T21:01:25.764 INFO:teuthology.orchestra.run.trial196.stdout:(3/34): libcephfs2-20.2.0-677.gf7870454.el9.x86 13 MB/s | 869 kB 00:00 2026-02-20T21:01:25.820 INFO:teuthology.orchestra.run.trial196.stdout:(4/34): libradosstriper1-20.2.0-677.gf7870454.e 4.4 MB/s | 250 kB 00:00 2026-02-20T21:01:25.912 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: ceph-selinux-2:20.2.0-677.gf7870454.el9.x86_64 33/36 2026-02-20T21:01:25.912 INFO:teuthology.orchestra.run.trial199.stdout:skipping the directory /sys 2026-02-20T21:01:25.912 INFO:teuthology.orchestra.run.trial199.stdout:skipping the directory /proc 2026-02-20T21:01:25.912 INFO:teuthology.orchestra.run.trial199.stdout:skipping the directory /mnt 2026-02-20T21:01:25.912 INFO:teuthology.orchestra.run.trial199.stdout:skipping the directory /var/tmp 2026-02-20T21:01:25.913 INFO:teuthology.orchestra.run.trial199.stdout:skipping the directory /home 2026-02-20T21:01:25.913 INFO:teuthology.orchestra.run.trial199.stdout:skipping the directory /root 2026-02-20T21:01:25.913 INFO:teuthology.orchestra.run.trial199.stdout:skipping the directory /tmp 2026-02-20T21:01:25.913 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:26.201 INFO:teuthology.orchestra.run.trial196.stdout:(5/34): librgw2-20.2.0-677.gf7870454.el9.x86_64 17 MB/s | 6.4 MB 00:00 2026-02-20T21:01:26.242 INFO:teuthology.orchestra.run.trial196.stdout:(6/34): python3-ceph-argparse-20.2.0-677.gf7870 1.1 MB/s | 45 kB 00:00 2026-02-20T21:01:26.286 INFO:teuthology.orchestra.run.trial196.stdout:(7/34): python3-ceph-common-20.2.0-677.gf787045 3.9 MB/s | 175 kB 00:00 2026-02-20T21:01:26.330 INFO:teuthology.orchestra.run.trial196.stdout:(8/34): python3-cephfs-20.2.0-677.gf7870454.el9 3.7 MB/s | 163 kB 00:00 2026-02-20T21:01:26.390 INFO:teuthology.orchestra.run.trial196.stdout:(9/34): ceph-common-20.2.0-677.gf7870454.el9.x8 22 MB/s | 24 MB 00:01 2026-02-20T21:01:26.391 INFO:teuthology.orchestra.run.trial196.stdout:(10/34): python3-rados-20.2.0-677.gf7870454.el9 5.1 MB/s | 323 kB 00:00 2026-02-20T21:01:26.416 INFO:teuthology.orchestra.run.trial196.stdout:(11/34): python3-rbd-20.2.0-677.gf7870454.el9.x 11 MB/s | 305 kB 00:00 2026-02-20T21:01:26.417 INFO:teuthology.orchestra.run.trial196.stdout:(12/34): python3-rgw-20.2.0-677.gf7870454.el9.x 3.7 MB/s | 99 kB 00:00 2026-02-20T21:01:26.466 INFO:teuthology.orchestra.run.trial196.stdout:(13/34): ceph-radosgw-20.2.0-677.gf7870454.el9. 20 MB/s | 24 MB 00:01 2026-02-20T21:01:26.534 INFO:teuthology.orchestra.run.trial196.stdout:(14/34): boost-program-options-1.75.0-13.el9.x8 1.5 MB/s | 104 kB 00:00 2026-02-20T21:01:26.577 INFO:teuthology.orchestra.run.trial196.stdout:(15/34): libnbd-1.20.3-4.el9.x86_64.rpm 3.7 MB/s | 164 kB 00:00 2026-02-20T21:01:26.601 INFO:teuthology.orchestra.run.trial196.stdout:(16/34): libpmemobj-1.12.1-1.el9.x86_64.rpm 6.5 MB/s | 160 kB 00:00 2026-02-20T21:01:26.623 INFO:teuthology.orchestra.run.trial196.stdout:(17/34): librabbitmq-0.11.0-7.el9.x86_64.rpm 2.0 MB/s | 45 kB 00:00 2026-02-20T21:01:26.667 INFO:teuthology.orchestra.run.trial196.stdout:(18/34): librdkafka-1.6.1-102.el9.x86_64.rpm 15 MB/s | 662 kB 00:00 2026-02-20T21:01:26.691 INFO:teuthology.orchestra.run.trial196.stdout:(19/34): lttng-ust-2.12.0-6.el9.x86_64.rpm 12 MB/s | 292 kB 00:00 2026-02-20T21:01:26.714 INFO:teuthology.orchestra.run.trial196.stdout:(20/34): qatlib-25.08.0-2.el9.x86_64.rpm 10 MB/s | 240 kB 00:00 2026-02-20T21:01:26.737 INFO:teuthology.orchestra.run.trial196.stdout:(21/34): qatlib-service-25.08.0-2.el9.x86_64.rp 1.7 MB/s | 37 kB 00:00 2026-02-20T21:01:26.758 INFO:teuthology.orchestra.run.trial196.stdout:(22/34): qatzip-libs-1.3.1-1.el9.x86_64.rpm 3.0 MB/s | 66 kB 00:00 2026-02-20T21:01:26.792 INFO:teuthology.orchestra.run.trial196.stdout:(23/34): gperftools-libs-2.9.1-3.el9.x86_64.rpm 8.9 MB/s | 308 kB 00:00 2026-02-20T21:01:26.813 INFO:teuthology.orchestra.run.trial196.stdout:(24/34): libarrow-9.0.0-15.el9.x86_64.rpm 217 MB/s | 4.4 MB 00:00 2026-02-20T21:01:26.816 INFO:teuthology.orchestra.run.trial196.stdout:(25/34): libarrow-doc-9.0.0-15.el9.noarch.rpm 7.5 MB/s | 25 kB 00:00 2026-02-20T21:01:26.819 INFO:teuthology.orchestra.run.trial196.stdout:(26/34): liboath-2.6.12-1.el9.x86_64.rpm 15 MB/s | 49 kB 00:00 2026-02-20T21:01:26.823 INFO:teuthology.orchestra.run.trial196.stdout:(27/34): libunwind-1.6.2-1.el9.x86_64.rpm 20 MB/s | 67 kB 00:00 2026-02-20T21:01:26.829 INFO:teuthology.orchestra.run.trial196.stdout:(28/34): parquet-libs-9.0.0-15.el9.x86_64.rpm 143 MB/s | 838 kB 00:00 2026-02-20T21:01:26.833 INFO:teuthology.orchestra.run.trial196.stdout:(29/34): re2-20211101-20.el9.x86_64.rpm 49 MB/s | 191 kB 00:00 2026-02-20T21:01:26.841 INFO:teuthology.orchestra.run.trial196.stdout:(30/34): thrift-0.15.0-4.el9.x86_64.rpm 201 MB/s | 1.6 MB 00:00 2026-02-20T21:01:26.908 INFO:teuthology.orchestra.run.trial196.stdout:(31/34): librados2-20.2.0-677.gf7870454.el9.x86 52 MB/s | 3.5 MB 00:00 2026-02-20T21:01:26.947 INFO:teuthology.orchestra.run.trial199.stdout: Installing : ceph-radosgw-2:20.2.0-677.gf7870454.el9.x86_64 34/36 2026-02-20T21:01:26.962 INFO:teuthology.orchestra.run.trial196.stdout:(32/34): librbd1-20.2.0-677.gf7870454.el9.x86_6 53 MB/s | 2.8 MB 00:00 2026-02-20T21:01:26.970 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: ceph-radosgw-2:20.2.0-677.gf7870454.el9.x86_64 34/36 2026-02-20T21:01:26.970 INFO:teuthology.orchestra.run.trial199.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:01:26.970 INFO:teuthology.orchestra.run.trial199.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-02-20T21:01:26.970 INFO:teuthology.orchestra.run.trial199.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-20T21:01:26.970 INFO:teuthology.orchestra.run.trial199.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-20T21:01:26.970 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:26.972 INFO:teuthology.orchestra.run.trial199.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 35/36 2026-02-20T21:01:26.975 INFO:teuthology.orchestra.run.trial196.stdout:(33/34): mailcap-2.1.49-5.el9.noarch.rpm 59 kB/s | 33 kB 00:00 2026-02-20T21:01:26.985 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 35/36 2026-02-20T21:01:26.985 INFO:teuthology.orchestra.run.trial199.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 36/36 2026-02-20T21:01:27.040 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 36/36 2026-02-20T21:01:27.040 INFO:teuthology.orchestra.run.trial199.stdout:Creating group 'qat' with GID 992. 2026-02-20T21:01:27.041 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:27.212 INFO:teuthology.orchestra.run.trial196.stdout:(34/34): pciutils-3.7.0-7.el9.x86_64.rpm 117 kB/s | 93 kB 00:00 2026-02-20T21:01:27.217 INFO:teuthology.orchestra.run.trial196.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:01:27.217 INFO:teuthology.orchestra.run.trial196.stdout:Total 32 MB/s | 78 MB 00:02 2026-02-20T21:01:27.306 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction check 2026-02-20T21:01:27.322 INFO:teuthology.orchestra.run.trial196.stdout:Transaction check succeeded. 2026-02-20T21:01:27.323 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction test 2026-02-20T21:01:27.382 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : ceph-base-2:20.2.0-677.gf7870454.el9.x86_64 1/36 2026-02-20T21:01:27.382 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 2/36 2026-02-20T21:01:27.383 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : ceph-radosgw-2:20.2.0-677.gf7870454.el9.x86_64 3/36 2026-02-20T21:01:27.383 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : ceph-selinux-2:20.2.0-677.gf7870454.el9.x86_64 4/36 2026-02-20T21:01:27.383 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : libcephfs2-2:20.2.0-677.gf7870454.el9.x86_64 5/36 2026-02-20T21:01:27.383 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : libradosstriper1-2:20.2.0-677.gf7870454.el9.x86_64 6/36 2026-02-20T21:01:27.383 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : librgw2-2:20.2.0-677.gf7870454.el9.x86_64 7/36 2026-02-20T21:01:27.383 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-ceph-argparse-2:20.2.0-677.gf7870454.el9.x 8/36 2026-02-20T21:01:27.383 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-ceph-common-2:20.2.0-677.gf7870454.el9.x86 9/36 2026-02-20T21:01:27.383 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-cephfs-2:20.2.0-677.gf7870454.el9.x86_64 10/36 2026-02-20T21:01:27.383 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-rados-2:20.2.0-677.gf7870454.el9.x86_64 11/36 2026-02-20T21:01:27.383 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-rbd-2:20.2.0-677.gf7870454.el9.x86_64 12/36 2026-02-20T21:01:27.383 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-rgw-2:20.2.0-677.gf7870454.el9.x86_64 13/36 2026-02-20T21:01:27.383 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 14/36 2026-02-20T21:01:27.383 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 15/36 2026-02-20T21:01:27.383 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 16/36 2026-02-20T21:01:27.383 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 17/36 2026-02-20T21:01:27.383 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 18/36 2026-02-20T21:01:27.384 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 19/36 2026-02-20T21:01:27.384 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 20/36 2026-02-20T21:01:27.384 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 21/36 2026-02-20T21:01:27.384 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 22/36 2026-02-20T21:01:27.384 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 23/36 2026-02-20T21:01:27.384 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 24/36 2026-02-20T21:01:27.384 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 25/36 2026-02-20T21:01:27.384 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 26/36 2026-02-20T21:01:27.384 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 27/36 2026-02-20T21:01:27.384 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 28/36 2026-02-20T21:01:27.384 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 29/36 2026-02-20T21:01:27.384 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 30/36 2026-02-20T21:01:27.384 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : re2-1:20211101-20.el9.x86_64 31/36 2026-02-20T21:01:27.385 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 32/36 2026-02-20T21:01:27.385 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : librados2-2:20.2.0-677.gf7870454.el9.x86_64 33/36 2026-02-20T21:01:27.385 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 34/36 2026-02-20T21:01:27.385 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : librbd1-2:20.2.0-677.gf7870454.el9.x86_64 35/36 2026-02-20T21:01:27.444 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 36/36 2026-02-20T21:01:27.444 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:27.445 INFO:teuthology.orchestra.run.trial199.stdout:Upgraded: 2026-02-20T21:01:27.445 INFO:teuthology.orchestra.run.trial199.stdout: librados2-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:27.445 INFO:teuthology.orchestra.run.trial199.stdout: librbd1-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:27.445 INFO:teuthology.orchestra.run.trial199.stdout:Installed: 2026-02-20T21:01:27.445 INFO:teuthology.orchestra.run.trial199.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-02-20T21:01:27.445 INFO:teuthology.orchestra.run.trial199.stdout: ceph-base-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:27.445 INFO:teuthology.orchestra.run.trial199.stdout: ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:27.445 INFO:teuthology.orchestra.run.trial199.stdout: ceph-radosgw-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:27.445 INFO:teuthology.orchestra.run.trial199.stdout: ceph-selinux-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:27.445 INFO:teuthology.orchestra.run.trial199.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-02-20T21:01:27.445 INFO:teuthology.orchestra.run.trial199.stdout: libarrow-9.0.0-15.el9.x86_64 2026-02-20T21:01:27.445 INFO:teuthology.orchestra.run.trial199.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-02-20T21:01:27.445 INFO:teuthology.orchestra.run.trial199.stdout: libcephfs2-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:27.445 INFO:teuthology.orchestra.run.trial199.stdout: libnbd-1.20.3-4.el9.x86_64 2026-02-20T21:01:27.445 INFO:teuthology.orchestra.run.trial199.stdout: liboath-2.6.12-1.el9.x86_64 2026-02-20T21:01:27.446 INFO:teuthology.orchestra.run.trial199.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-02-20T21:01:27.446 INFO:teuthology.orchestra.run.trial199.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-02-20T21:01:27.446 INFO:teuthology.orchestra.run.trial199.stdout: libradosstriper1-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:27.446 INFO:teuthology.orchestra.run.trial199.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-02-20T21:01:27.446 INFO:teuthology.orchestra.run.trial199.stdout: librgw2-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:27.446 INFO:teuthology.orchestra.run.trial199.stdout: libunwind-1.6.2-1.el9.x86_64 2026-02-20T21:01:27.446 INFO:teuthology.orchestra.run.trial199.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-02-20T21:01:27.446 INFO:teuthology.orchestra.run.trial199.stdout: mailcap-2.1.49-5.el9.noarch 2026-02-20T21:01:27.446 INFO:teuthology.orchestra.run.trial199.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-02-20T21:01:27.446 INFO:teuthology.orchestra.run.trial199.stdout: pciutils-3.7.0-7.el9.x86_64 2026-02-20T21:01:27.446 INFO:teuthology.orchestra.run.trial199.stdout: python3-ceph-argparse-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:27.446 INFO:teuthology.orchestra.run.trial199.stdout: python3-ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:27.446 INFO:teuthology.orchestra.run.trial199.stdout: python3-cephfs-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:27.446 INFO:teuthology.orchestra.run.trial199.stdout: python3-rados-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:27.446 INFO:teuthology.orchestra.run.trial199.stdout: python3-rbd-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:27.446 INFO:teuthology.orchestra.run.trial199.stdout: python3-rgw-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:27.446 INFO:teuthology.orchestra.run.trial199.stdout: qatlib-25.08.0-2.el9.x86_64 2026-02-20T21:01:27.447 INFO:teuthology.orchestra.run.trial199.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-02-20T21:01:27.447 INFO:teuthology.orchestra.run.trial199.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-02-20T21:01:27.447 INFO:teuthology.orchestra.run.trial199.stdout: re2-1:20211101-20.el9.x86_64 2026-02-20T21:01:27.447 INFO:teuthology.orchestra.run.trial199.stdout: thrift-0.15.0-4.el9.x86_64 2026-02-20T21:01:27.447 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:27.447 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:01:27.487 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install ceph-test 2026-02-20T21:01:27.618 INFO:teuthology.orchestra.run.trial196.stdout:Transaction test succeeded. 2026-02-20T21:01:27.618 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction 2026-02-20T21:01:27.799 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:12 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:01:27.881 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:01:27.882 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:27.882 INFO:teuthology.orchestra.run.trial199.stdout: Package Arch Version Repository Size 2026-02-20T21:01:27.882 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:27.882 INFO:teuthology.orchestra.run.trial199.stdout:Installing: 2026-02-20T21:01:27.883 INFO:teuthology.orchestra.run.trial199.stdout: ceph-test x86_64 2:20.2.0-677.gf7870454.el9 ceph 84 M 2026-02-20T21:01:27.883 INFO:teuthology.orchestra.run.trial199.stdout:Installing dependencies: 2026-02-20T21:01:27.883 INFO:teuthology.orchestra.run.trial199.stdout: libcephsqlite x86_64 2:20.2.0-677.gf7870454.el9 ceph 164 k 2026-02-20T21:01:27.883 INFO:teuthology.orchestra.run.trial199.stdout: libxslt x86_64 1.1.34-12.el9 appstream 233 k 2026-02-20T21:01:27.883 INFO:teuthology.orchestra.run.trial199.stdout: socat x86_64 1.7.4.1-8.el9 appstream 303 k 2026-02-20T21:01:27.883 INFO:teuthology.orchestra.run.trial199.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2026-02-20T21:01:27.883 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:27.883 INFO:teuthology.orchestra.run.trial199.stdout:Transaction Summary 2026-02-20T21:01:27.883 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:27.883 INFO:teuthology.orchestra.run.trial199.stdout:Install 5 Packages 2026-02-20T21:01:27.883 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:27.884 INFO:teuthology.orchestra.run.trial199.stdout:Total download size: 85 M 2026-02-20T21:01:27.884 INFO:teuthology.orchestra.run.trial199.stdout:Installed size: 365 M 2026-02-20T21:01:27.884 INFO:teuthology.orchestra.run.trial199.stdout:Downloading Packages: 2026-02-20T21:01:27.949 INFO:teuthology.orchestra.run.trial196.stdout: Preparing : 1/1 2026-02-20T21:01:27.963 INFO:teuthology.orchestra.run.trial196.stdout: Installing : thrift-0.15.0-4.el9.x86_64 1/36 2026-02-20T21:01:27.992 INFO:teuthology.orchestra.run.trial199.stdout:(1/5): libxslt-1.1.34-12.el9.x86_64.rpm 9.1 MB/s | 233 kB 00:00 2026-02-20T21:01:28.004 INFO:teuthology.orchestra.run.trial199.stdout:(2/5): socat-1.7.4.1-8.el9.x86_64.rpm 25 MB/s | 303 kB 00:00 2026-02-20T21:01:28.009 INFO:teuthology.orchestra.run.trial199.stdout:(3/5): xmlstarlet-1.6.1-20.el9.x86_64.rpm 14 MB/s | 64 kB 00:00 2026-02-20T21:01:28.055 INFO:teuthology.orchestra.run.trial199.stdout:(4/5): libcephsqlite-20.2.0-677.gf7870454.el9.x 1.8 MB/s | 164 kB 00:00 2026-02-20T21:01:28.126 INFO:teuthology.orchestra.run.trial196.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 2/36 2026-02-20T21:01:28.131 INFO:teuthology.orchestra.run.trial196.stdout: Upgrading : librados2-2:20.2.0-677.gf7870454.el9.x86_64 3/36 2026-02-20T21:01:28.158 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: librados2-2:20.2.0-677.gf7870454.el9.x86_64 3/36 2026-02-20T21:01:28.199 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-rados-2:20.2.0-677.gf7870454.el9.x86_64 4/36 2026-02-20T21:01:28.200 INFO:teuthology.orchestra.run.trial196.stdout: Installing : libcephfs2-2:20.2.0-677.gf7870454.el9.x86_64 5/36 2026-02-20T21:01:28.211 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: libcephfs2-2:20.2.0-677.gf7870454.el9.x86_64 5/36 2026-02-20T21:01:28.219 INFO:teuthology.orchestra.run.trial196.stdout: Installing : liboath-2.6.12-1.el9.x86_64 6/36 2026-02-20T21:01:28.222 INFO:teuthology.orchestra.run.trial196.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 7/36 2026-02-20T21:01:28.225 INFO:teuthology.orchestra.run.trial196.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 8/36 2026-02-20T21:01:28.229 INFO:teuthology.orchestra.run.trial196.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 9/36 2026-02-20T21:01:28.238 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-ceph-argparse-2:20.2.0-677.gf7870454.el9.x 10/36 2026-02-20T21:01:28.250 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-cephfs-2:20.2.0-677.gf7870454.el9.x86_64 11/36 2026-02-20T21:01:28.251 INFO:teuthology.orchestra.run.trial196.stdout: Installing : libradosstriper1-2:20.2.0-677.gf7870454.el9.x86_64 12/36 2026-02-20T21:01:28.263 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: libradosstriper1-2:20.2.0-677.gf7870454.el9.x86_64 12/36 2026-02-20T21:01:28.293 INFO:teuthology.orchestra.run.trial196.stdout: Installing : re2-1:20211101-20.el9.x86_64 13/36 2026-02-20T21:01:28.297 INFO:teuthology.orchestra.run.trial196.stdout: Installing : libarrow-9.0.0-15.el9.x86_64 14/36 2026-02-20T21:01:28.303 INFO:teuthology.orchestra.run.trial196.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 15/36 2026-02-20T21:01:28.306 INFO:teuthology.orchestra.run.trial196.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 16/36 2026-02-20T21:01:28.311 INFO:teuthology.orchestra.run.trial196.stdout: Installing : libarrow-doc-9.0.0-15.el9.noarch 17/36 2026-02-20T21:01:28.440 INFO:teuthology.orchestra.run.trial196.stdout: Installing : libnbd-1.20.3-4.el9.x86_64 18/36 2026-02-20T21:01:28.441 INFO:teuthology.orchestra.run.trial196.stdout: Upgrading : librbd1-2:20.2.0-677.gf7870454.el9.x86_64 19/36 2026-02-20T21:01:28.467 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: librbd1-2:20.2.0-677.gf7870454.el9.x86_64 19/36 2026-02-20T21:01:28.469 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-rbd-2:20.2.0-677.gf7870454.el9.x86_64 20/36 2026-02-20T21:01:28.475 INFO:teuthology.orchestra.run.trial196.stdout: Installing : boost-program-options-1.75.0-13.el9.x86_64 21/36 2026-02-20T21:01:28.761 INFO:teuthology.orchestra.run.trial196.stdout: Installing : parquet-libs-9.0.0-15.el9.x86_64 22/36 2026-02-20T21:01:28.762 INFO:teuthology.orchestra.run.trial196.stdout: Installing : librgw2-2:20.2.0-677.gf7870454.el9.x86_64 23/36 2026-02-20T21:01:28.778 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: librgw2-2:20.2.0-677.gf7870454.el9.x86_64 23/36 2026-02-20T21:01:28.782 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-rgw-2:20.2.0-677.gf7870454.el9.x86_64 24/36 2026-02-20T21:01:28.787 INFO:teuthology.orchestra.run.trial196.stdout: Installing : pciutils-3.7.0-7.el9.x86_64 25/36 2026-02-20T21:01:28.789 INFO:teuthology.orchestra.run.trial196.stdout: Installing : qatlib-25.08.0-2.el9.x86_64 26/36 2026-02-20T21:01:28.790 INFO:teuthology.orchestra.run.trial196.stdout: Installing : qatlib-service-25.08.0-2.el9.x86_64 27/36 2026-02-20T21:01:28.808 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 27/36 2026-02-20T21:01:28.812 INFO:teuthology.orchestra.run.trial196.stdout: Installing : qatzip-libs-1.3.1-1.el9.x86_64 28/36 2026-02-20T21:01:28.845 INFO:teuthology.orchestra.run.trial196.stdout: Installing : mailcap-2.1.49-5.el9.noarch 29/36 2026-02-20T21:01:28.847 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-ceph-common-2:20.2.0-677.gf7870454.el9.x86 30/36 2026-02-20T21:01:29.667 INFO:teuthology.orchestra.run.trial199.stdout:(5/5): ceph-test-20.2.0-677.gf7870454.el9.x86_6 49 MB/s | 84 MB 00:01 2026-02-20T21:01:29.669 INFO:teuthology.orchestra.run.trial199.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:01:29.669 INFO:teuthology.orchestra.run.trial199.stdout:Total 47 MB/s | 85 MB 00:01 2026-02-20T21:01:29.697 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction check 2026-02-20T21:01:29.707 INFO:teuthology.orchestra.run.trial199.stdout:Transaction check succeeded. 2026-02-20T21:01:29.707 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction test 2026-02-20T21:01:29.963 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 31/36 2026-02-20T21:01:29.966 INFO:teuthology.orchestra.run.trial196.stdout: Installing : ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 31/36 2026-02-20T21:01:29.995 INFO:teuthology.orchestra.run.trial199.stdout:Transaction test succeeded. 2026-02-20T21:01:29.995 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction 2026-02-20T21:01:30.275 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 31/36 2026-02-20T21:01:30.278 INFO:teuthology.orchestra.run.trial196.stdout: Installing : ceph-base-2:20.2.0-677.gf7870454.el9.x86_64 32/36 2026-02-20T21:01:30.309 INFO:teuthology.orchestra.run.trial199.stdout: Preparing : 1/1 2026-02-20T21:01:30.315 INFO:teuthology.orchestra.run.trial199.stdout: Installing : socat-1.7.4.1-8.el9.x86_64 1/5 2026-02-20T21:01:30.318 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: ceph-base-2:20.2.0-677.gf7870454.el9.x86_64 32/36 2026-02-20T21:01:30.318 INFO:teuthology.orchestra.run.trial196.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2026-02-20T21:01:30.319 INFO:teuthology.orchestra.run.trial196.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2026-02-20T21:01:30.319 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:30.320 INFO:teuthology.orchestra.run.trial199.stdout: Installing : libxslt-1.1.34-12.el9.x86_64 2/5 2026-02-20T21:01:30.322 INFO:teuthology.orchestra.run.trial196.stdout: Installing : ceph-selinux-2:20.2.0-677.gf7870454.el9.x86_64 33/36 2026-02-20T21:01:30.328 INFO:teuthology.orchestra.run.trial199.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 3/5 2026-02-20T21:01:30.329 INFO:teuthology.orchestra.run.trial199.stdout: Installing : libcephsqlite-2:20.2.0-677.gf7870454.el9.x86_64 4/5 2026-02-20T21:01:34.045 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: libcephsqlite-2:20.2.0-677.gf7870454.el9.x86_64 4/5 2026-02-20T21:01:34.063 INFO:teuthology.orchestra.run.trial199.stdout: Installing : ceph-test-2:20.2.0-677.gf7870454.el9.x86_64 5/5 2026-02-20T21:01:34.392 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: ceph-test-2:20.2.0-677.gf7870454.el9.x86_64 5/5 2026-02-20T21:01:34.392 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : ceph-test-2:20.2.0-677.gf7870454.el9.x86_64 1/5 2026-02-20T21:01:34.392 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : libcephsqlite-2:20.2.0-677.gf7870454.el9.x86_64 2/5 2026-02-20T21:01:34.392 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 3/5 2026-02-20T21:01:34.392 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 4/5 2026-02-20T21:01:34.444 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/5 2026-02-20T21:01:34.445 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:34.445 INFO:teuthology.orchestra.run.trial199.stdout:Installed: 2026-02-20T21:01:34.445 INFO:teuthology.orchestra.run.trial199.stdout: ceph-test-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:34.445 INFO:teuthology.orchestra.run.trial199.stdout: libcephsqlite-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:34.445 INFO:teuthology.orchestra.run.trial199.stdout: libxslt-1.1.34-12.el9.x86_64 2026-02-20T21:01:34.445 INFO:teuthology.orchestra.run.trial199.stdout: socat-1.7.4.1-8.el9.x86_64 2026-02-20T21:01:34.445 INFO:teuthology.orchestra.run.trial199.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2026-02-20T21:01:34.445 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:34.445 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:01:34.482 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install ceph 2026-02-20T21:01:34.788 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:19 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:01:34.845 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: ceph-selinux-2:20.2.0-677.gf7870454.el9.x86_64 33/36 2026-02-20T21:01:34.845 INFO:teuthology.orchestra.run.trial196.stdout:skipping the directory /sys 2026-02-20T21:01:34.845 INFO:teuthology.orchestra.run.trial196.stdout:skipping the directory /proc 2026-02-20T21:01:34.845 INFO:teuthology.orchestra.run.trial196.stdout:skipping the directory /mnt 2026-02-20T21:01:34.845 INFO:teuthology.orchestra.run.trial196.stdout:skipping the directory /var/tmp 2026-02-20T21:01:34.845 INFO:teuthology.orchestra.run.trial196.stdout:skipping the directory /home 2026-02-20T21:01:34.845 INFO:teuthology.orchestra.run.trial196.stdout:skipping the directory /root 2026-02-20T21:01:34.845 INFO:teuthology.orchestra.run.trial196.stdout:skipping the directory /tmp 2026-02-20T21:01:34.846 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:34.871 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:01:34.873 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:34.873 INFO:teuthology.orchestra.run.trial199.stdout: Package Arch Version Repository Size 2026-02-20T21:01:34.873 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:34.873 INFO:teuthology.orchestra.run.trial199.stdout:Installing: 2026-02-20T21:01:34.873 INFO:teuthology.orchestra.run.trial199.stdout: ceph x86_64 2:20.2.0-677.gf7870454.el9 ceph 6.5 k 2026-02-20T21:01:34.873 INFO:teuthology.orchestra.run.trial199.stdout:Installing dependencies: 2026-02-20T21:01:34.873 INFO:teuthology.orchestra.run.trial199.stdout: ceph-mds x86_64 2:20.2.0-677.gf7870454.el9 ceph 2.3 M 2026-02-20T21:01:34.873 INFO:teuthology.orchestra.run.trial199.stdout: ceph-mgr x86_64 2:20.2.0-677.gf7870454.el9 ceph 961 k 2026-02-20T21:01:34.873 INFO:teuthology.orchestra.run.trial199.stdout: ceph-mgr-modules-core noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 290 k 2026-02-20T21:01:34.874 INFO:teuthology.orchestra.run.trial199.stdout: ceph-mon x86_64 2:20.2.0-677.gf7870454.el9 ceph 5.0 M 2026-02-20T21:01:34.874 INFO:teuthology.orchestra.run.trial199.stdout: ceph-osd x86_64 2:20.2.0-677.gf7870454.el9 ceph 17 M 2026-02-20T21:01:34.874 INFO:teuthology.orchestra.run.trial199.stdout: ledmon-libs x86_64 1.1.0-3.el9 baseos 40 k 2026-02-20T21:01:34.874 INFO:teuthology.orchestra.run.trial199.stdout: libconfig x86_64 1.7.2-9.el9 baseos 72 k 2026-02-20T21:01:34.874 INFO:teuthology.orchestra.run.trial199.stdout: libstoragemgmt x86_64 1.10.1-1.el9 appstream 246 k 2026-02-20T21:01:34.874 INFO:teuthology.orchestra.run.trial199.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2026-02-20T21:01:34.874 INFO:teuthology.orchestra.run.trial199.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2026-02-20T21:01:34.874 INFO:teuthology.orchestra.run.trial199.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2026-02-20T21:01:34.874 INFO:teuthology.orchestra.run.trial199.stdout: python3-autocommand noarch 2.2.2-8.el9 epel 29 k 2026-02-20T21:01:34.874 INFO:teuthology.orchestra.run.trial199.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 epel 60 k 2026-02-20T21:01:34.874 INFO:teuthology.orchestra.run.trial199.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2026-02-20T21:01:34.875 INFO:teuthology.orchestra.run.trial199.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2026-02-20T21:01:34.875 INFO:teuthology.orchestra.run.trial199.stdout: python3-cheroot noarch 10.0.1-4.el9 epel 173 k 2026-02-20T21:01:34.875 INFO:teuthology.orchestra.run.trial199.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2026-02-20T21:01:34.875 INFO:teuthology.orchestra.run.trial199.stdout: python3-cryptography x86_64 36.0.1-5.el9 baseos 1.2 M 2026-02-20T21:01:34.875 INFO:teuthology.orchestra.run.trial199.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2026-02-20T21:01:34.875 INFO:teuthology.orchestra.run.trial199.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2026-02-20T21:01:34.875 INFO:teuthology.orchestra.run.trial199.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2026-02-20T21:01:34.875 INFO:teuthology.orchestra.run.trial199.stdout: python3-jaraco-context noarch 6.0.1-3.el9 epel 20 k 2026-02-20T21:01:34.875 INFO:teuthology.orchestra.run.trial199.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2026-02-20T21:01:34.875 INFO:teuthology.orchestra.run.trial199.stdout: python3-jaraco-text noarch 4.0.0-2.el9 epel 26 k 2026-02-20T21:01:34.875 INFO:teuthology.orchestra.run.trial199.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 appstream 177 k 2026-02-20T21:01:34.875 INFO:teuthology.orchestra.run.trial199.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2026-02-20T21:01:34.876 INFO:teuthology.orchestra.run.trial199.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2026-02-20T21:01:34.876 INFO:teuthology.orchestra.run.trial199.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2026-02-20T21:01:34.876 INFO:teuthology.orchestra.run.trial199.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2026-02-20T21:01:34.876 INFO:teuthology.orchestra.run.trial199.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2026-02-20T21:01:34.876 INFO:teuthology.orchestra.run.trial199.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2026-02-20T21:01:34.876 INFO:teuthology.orchestra.run.trial199.stdout: python3-pyparsing noarch 2.4.7-9.el9 baseos 150 k 2026-02-20T21:01:34.876 INFO:teuthology.orchestra.run.trial199.stdout: python3-requests noarch 2.25.1-10.el9 baseos 126 k 2026-02-20T21:01:34.876 INFO:teuthology.orchestra.run.trial199.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2026-02-20T21:01:34.876 INFO:teuthology.orchestra.run.trial199.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2026-02-20T21:01:34.876 INFO:teuthology.orchestra.run.trial199.stdout: python3-urllib3 noarch 1.26.5-7.el9 baseos 218 k 2026-02-20T21:01:34.876 INFO:teuthology.orchestra.run.trial199.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2026-02-20T21:01:34.876 INFO:teuthology.orchestra.run.trial199.stdout: unzip x86_64 6.0-59.el9 baseos 182 k 2026-02-20T21:01:34.876 INFO:teuthology.orchestra.run.trial199.stdout: zip x86_64 3.0-35.el9 baseos 266 k 2026-02-20T21:01:34.877 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:34.877 INFO:teuthology.orchestra.run.trial199.stdout:Transaction Summary 2026-02-20T21:01:34.877 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:34.877 INFO:teuthology.orchestra.run.trial199.stdout:Install 40 Packages 2026-02-20T21:01:34.877 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:34.878 INFO:teuthology.orchestra.run.trial199.stdout:Total download size: 30 M 2026-02-20T21:01:34.878 INFO:teuthology.orchestra.run.trial199.stdout:Installed size: 100 M 2026-02-20T21:01:34.878 INFO:teuthology.orchestra.run.trial199.stdout:Downloading Packages: 2026-02-20T21:01:35.510 INFO:teuthology.orchestra.run.trial199.stdout:(1/40): ceph-20.2.0-677.gf7870454.el9.x86_64.rp 127 kB/s | 6.5 kB 00:00 2026-02-20T21:01:35.595 INFO:teuthology.orchestra.run.trial199.stdout:(2/40): ceph-mgr-20.2.0-677.gf7870454.el9.x86_6 6.9 MB/s | 961 kB 00:00 2026-02-20T21:01:35.626 INFO:teuthology.orchestra.run.trial199.stdout:(3/40): ceph-mds-20.2.0-677.gf7870454.el9.x86_6 14 MB/s | 2.3 MB 00:00 2026-02-20T21:01:35.662 INFO:teuthology.orchestra.run.trial199.stdout:(4/40): ceph-mgr-modules-core-20.2.0-677.gf7870 8.0 MB/s | 290 kB 00:00 2026-02-20T21:01:35.724 INFO:teuthology.orchestra.run.trial199.stdout:(5/40): ledmon-libs-1.1.0-3.el9.x86_64.rpm 658 kB/s | 40 kB 00:00 2026-02-20T21:01:35.754 INFO:teuthology.orchestra.run.trial199.stdout:(6/40): ceph-mon-20.2.0-677.gf7870454.el9.x86_6 21 MB/s | 5.0 MB 00:00 2026-02-20T21:01:35.775 INFO:teuthology.orchestra.run.trial199.stdout:(7/40): libconfig-1.7.2-9.el9.x86_64.rpm 1.4 MB/s | 72 kB 00:00 2026-02-20T21:01:35.838 INFO:teuthology.orchestra.run.trial199.stdout:(8/40): python3-cryptography-36.0.1-5.el9.x86_6 20 MB/s | 1.2 MB 00:00 2026-02-20T21:01:35.864 INFO:teuthology.orchestra.run.trial199.stdout:(9/40): python3-cffi-1.14.5-5.el9.x86_64.rpm 2.2 MB/s | 253 kB 00:00 2026-02-20T21:01:35.870 INFO:teuthology.orchestra.run.trial199.stdout:(10/40): python3-ply-3.11-14.el9.noarch.rpm 3.2 MB/s | 106 kB 00:00 2026-02-20T21:01:35.897 INFO:teuthology.orchestra.run.trial199.stdout:(11/40): python3-pyparsing-2.4.7-9.el9.noarch.r 5.5 MB/s | 150 kB 00:00 2026-02-20T21:01:35.898 INFO:teuthology.orchestra.run.trial199.stdout:(12/40): python3-pycparser-2.20-6.el9.noarch.rp 3.9 MB/s | 135 kB 00:00 2026-02-20T21:01:35.922 INFO:teuthology.orchestra.run.trial196.stdout: Installing : ceph-radosgw-2:20.2.0-677.gf7870454.el9.x86_64 34/36 2026-02-20T21:01:35.928 INFO:teuthology.orchestra.run.trial199.stdout:(13/40): python3-requests-2.25.1-10.el9.noarch. 4.0 MB/s | 126 kB 00:00 2026-02-20T21:01:35.940 INFO:teuthology.orchestra.run.trial199.stdout:(14/40): python3-urllib3-1.26.5-7.el9.noarch.rp 5.1 MB/s | 218 kB 00:00 2026-02-20T21:01:35.945 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: ceph-radosgw-2:20.2.0-677.gf7870454.el9.x86_64 34/36 2026-02-20T21:01:35.945 INFO:teuthology.orchestra.run.trial196.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:01:35.946 INFO:teuthology.orchestra.run.trial196.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-02-20T21:01:35.946 INFO:teuthology.orchestra.run.trial196.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-20T21:01:35.946 INFO:teuthology.orchestra.run.trial196.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-20T21:01:35.946 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:35.947 INFO:teuthology.orchestra.run.trial196.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 35/36 2026-02-20T21:01:35.959 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 35/36 2026-02-20T21:01:35.960 INFO:teuthology.orchestra.run.trial196.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 36/36 2026-02-20T21:01:35.964 INFO:teuthology.orchestra.run.trial199.stdout:(15/40): unzip-6.0-59.el9.x86_64.rpm 5.0 MB/s | 182 kB 00:00 2026-02-20T21:01:36.001 INFO:teuthology.orchestra.run.trial199.stdout:(16/40): ceph-osd-20.2.0-677.gf7870454.el9.x86_ 42 MB/s | 17 MB 00:00 2026-02-20T21:01:36.002 INFO:teuthology.orchestra.run.trial199.stdout:(17/40): zip-3.0-35.el9.x86_64.rpm 4.2 MB/s | 266 kB 00:00 2026-02-20T21:01:36.016 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 36/36 2026-02-20T21:01:36.016 INFO:teuthology.orchestra.run.trial196.stdout:Creating group 'qat' with GID 992. 2026-02-20T21:01:36.016 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:36.362 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : ceph-base-2:20.2.0-677.gf7870454.el9.x86_64 1/36 2026-02-20T21:01:36.362 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 2/36 2026-02-20T21:01:36.362 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : ceph-radosgw-2:20.2.0-677.gf7870454.el9.x86_64 3/36 2026-02-20T21:01:36.362 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : ceph-selinux-2:20.2.0-677.gf7870454.el9.x86_64 4/36 2026-02-20T21:01:36.362 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : libcephfs2-2:20.2.0-677.gf7870454.el9.x86_64 5/36 2026-02-20T21:01:36.362 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : libradosstriper1-2:20.2.0-677.gf7870454.el9.x86_64 6/36 2026-02-20T21:01:36.362 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : librgw2-2:20.2.0-677.gf7870454.el9.x86_64 7/36 2026-02-20T21:01:36.362 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-ceph-argparse-2:20.2.0-677.gf7870454.el9.x 8/36 2026-02-20T21:01:36.363 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-ceph-common-2:20.2.0-677.gf7870454.el9.x86 9/36 2026-02-20T21:01:36.363 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-cephfs-2:20.2.0-677.gf7870454.el9.x86_64 10/36 2026-02-20T21:01:36.363 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-rados-2:20.2.0-677.gf7870454.el9.x86_64 11/36 2026-02-20T21:01:36.363 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-rbd-2:20.2.0-677.gf7870454.el9.x86_64 12/36 2026-02-20T21:01:36.363 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-rgw-2:20.2.0-677.gf7870454.el9.x86_64 13/36 2026-02-20T21:01:36.363 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 14/36 2026-02-20T21:01:36.363 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 15/36 2026-02-20T21:01:36.363 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 16/36 2026-02-20T21:01:36.363 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 17/36 2026-02-20T21:01:36.363 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 18/36 2026-02-20T21:01:36.363 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 19/36 2026-02-20T21:01:36.363 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 20/36 2026-02-20T21:01:36.364 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 21/36 2026-02-20T21:01:36.364 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 22/36 2026-02-20T21:01:36.364 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 23/36 2026-02-20T21:01:36.364 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 24/36 2026-02-20T21:01:36.364 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 25/36 2026-02-20T21:01:36.364 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 26/36 2026-02-20T21:01:36.364 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 27/36 2026-02-20T21:01:36.364 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 28/36 2026-02-20T21:01:36.364 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 29/36 2026-02-20T21:01:36.364 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 30/36 2026-02-20T21:01:36.364 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : re2-1:20211101-20.el9.x86_64 31/36 2026-02-20T21:01:36.364 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 32/36 2026-02-20T21:01:36.364 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : librados2-2:20.2.0-677.gf7870454.el9.x86_64 33/36 2026-02-20T21:01:36.365 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 34/36 2026-02-20T21:01:36.365 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : librbd1-2:20.2.0-677.gf7870454.el9.x86_64 35/36 2026-02-20T21:01:36.423 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 36/36 2026-02-20T21:01:36.423 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:36.423 INFO:teuthology.orchestra.run.trial196.stdout:Upgraded: 2026-02-20T21:01:36.423 INFO:teuthology.orchestra.run.trial196.stdout: librados2-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:36.424 INFO:teuthology.orchestra.run.trial196.stdout: librbd1-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:36.424 INFO:teuthology.orchestra.run.trial196.stdout:Installed: 2026-02-20T21:01:36.424 INFO:teuthology.orchestra.run.trial196.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-02-20T21:01:36.424 INFO:teuthology.orchestra.run.trial196.stdout: ceph-base-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:36.424 INFO:teuthology.orchestra.run.trial196.stdout: ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:36.424 INFO:teuthology.orchestra.run.trial196.stdout: ceph-radosgw-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:36.424 INFO:teuthology.orchestra.run.trial196.stdout: ceph-selinux-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:36.424 INFO:teuthology.orchestra.run.trial196.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-02-20T21:01:36.424 INFO:teuthology.orchestra.run.trial196.stdout: libarrow-9.0.0-15.el9.x86_64 2026-02-20T21:01:36.424 INFO:teuthology.orchestra.run.trial196.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-02-20T21:01:36.424 INFO:teuthology.orchestra.run.trial196.stdout: libcephfs2-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:36.424 INFO:teuthology.orchestra.run.trial196.stdout: libnbd-1.20.3-4.el9.x86_64 2026-02-20T21:01:36.425 INFO:teuthology.orchestra.run.trial196.stdout: liboath-2.6.12-1.el9.x86_64 2026-02-20T21:01:36.425 INFO:teuthology.orchestra.run.trial196.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-02-20T21:01:36.425 INFO:teuthology.orchestra.run.trial196.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-02-20T21:01:36.425 INFO:teuthology.orchestra.run.trial196.stdout: libradosstriper1-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:36.425 INFO:teuthology.orchestra.run.trial196.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-02-20T21:01:36.425 INFO:teuthology.orchestra.run.trial196.stdout: librgw2-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:36.425 INFO:teuthology.orchestra.run.trial196.stdout: libunwind-1.6.2-1.el9.x86_64 2026-02-20T21:01:36.425 INFO:teuthology.orchestra.run.trial196.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-02-20T21:01:36.425 INFO:teuthology.orchestra.run.trial196.stdout: mailcap-2.1.49-5.el9.noarch 2026-02-20T21:01:36.425 INFO:teuthology.orchestra.run.trial196.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-02-20T21:01:36.425 INFO:teuthology.orchestra.run.trial196.stdout: pciutils-3.7.0-7.el9.x86_64 2026-02-20T21:01:36.425 INFO:teuthology.orchestra.run.trial196.stdout: python3-ceph-argparse-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:36.425 INFO:teuthology.orchestra.run.trial196.stdout: python3-ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:36.426 INFO:teuthology.orchestra.run.trial196.stdout: python3-cephfs-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:36.426 INFO:teuthology.orchestra.run.trial196.stdout: python3-rados-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:36.426 INFO:teuthology.orchestra.run.trial196.stdout: python3-rbd-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:36.426 INFO:teuthology.orchestra.run.trial196.stdout: python3-rgw-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:36.426 INFO:teuthology.orchestra.run.trial196.stdout: qatlib-25.08.0-2.el9.x86_64 2026-02-20T21:01:36.426 INFO:teuthology.orchestra.run.trial196.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-02-20T21:01:36.426 INFO:teuthology.orchestra.run.trial196.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-02-20T21:01:36.426 INFO:teuthology.orchestra.run.trial196.stdout: re2-1:20211101-20.el9.x86_64 2026-02-20T21:01:36.426 INFO:teuthology.orchestra.run.trial196.stdout: thrift-0.15.0-4.el9.x86_64 2026-02-20T21:01:36.426 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:36.426 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:01:36.467 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install ceph-test 2026-02-20T21:01:36.642 INFO:teuthology.orchestra.run.trial199.stdout:(18/40): lua-5.4.4-4.el9.x86_64.rpm 294 kB/s | 188 kB 00:00 2026-02-20T21:01:36.645 INFO:teuthology.orchestra.run.trial199.stdout:(19/40): python3-libstoragemgmt-1.10.1-1.el9.x8 276 kB/s | 177 kB 00:00 2026-02-20T21:01:36.696 INFO:teuthology.orchestra.run.trial199.stdout:(20/40): libstoragemgmt-1.10.1-1.el9.x86_64.rpm 337 kB/s | 246 kB 00:00 2026-02-20T21:01:36.728 INFO:teuthology.orchestra.run.trial199.stdout:(21/40): python3-toml-0.10.2-6.el9.noarch.rpm 498 kB/s | 42 kB 00:00 2026-02-20T21:01:36.775 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:13 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:01:36.805 INFO:teuthology.orchestra.run.trial199.stdout:(22/40): python3-packaging-20.9-5.el9.noarch.rp 474 kB/s | 77 kB 00:00 2026-02-20T21:01:36.861 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:01:36.862 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:01:36.862 INFO:teuthology.orchestra.run.trial196.stdout: Package Arch Version Repository Size 2026-02-20T21:01:36.862 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:01:36.862 INFO:teuthology.orchestra.run.trial196.stdout:Installing: 2026-02-20T21:01:36.862 INFO:teuthology.orchestra.run.trial196.stdout: ceph-test x86_64 2:20.2.0-677.gf7870454.el9 ceph 84 M 2026-02-20T21:01:36.862 INFO:teuthology.orchestra.run.trial196.stdout:Installing dependencies: 2026-02-20T21:01:36.862 INFO:teuthology.orchestra.run.trial196.stdout: libcephsqlite x86_64 2:20.2.0-677.gf7870454.el9 ceph 164 k 2026-02-20T21:01:36.862 INFO:teuthology.orchestra.run.trial196.stdout: libxslt x86_64 1.1.34-12.el9 appstream 233 k 2026-02-20T21:01:36.862 INFO:teuthology.orchestra.run.trial196.stdout: socat x86_64 1.7.4.1-8.el9 appstream 303 k 2026-02-20T21:01:36.862 INFO:teuthology.orchestra.run.trial196.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2026-02-20T21:01:36.862 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:36.862 INFO:teuthology.orchestra.run.trial196.stdout:Transaction Summary 2026-02-20T21:01:36.862 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:01:36.862 INFO:teuthology.orchestra.run.trial196.stdout:Install 5 Packages 2026-02-20T21:01:36.862 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:36.863 INFO:teuthology.orchestra.run.trial196.stdout:Total download size: 85 M 2026-02-20T21:01:36.863 INFO:teuthology.orchestra.run.trial196.stdout:Installed size: 365 M 2026-02-20T21:01:36.863 INFO:teuthology.orchestra.run.trial196.stdout:Downloading Packages: 2026-02-20T21:01:36.917 INFO:teuthology.orchestra.run.trial199.stdout:(23/40): python3-autocommand-2.2.2-8.el9.noarch 261 kB/s | 29 kB 00:00 2026-02-20T21:01:36.923 INFO:teuthology.orchestra.run.trial199.stdout:(24/40): luarocks-3.9.2-5.el9.noarch.rpm 779 kB/s | 151 kB 00:00 2026-02-20T21:01:37.015 INFO:teuthology.orchestra.run.trial199.stdout:(25/40): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 471 kB/s | 43 kB 00:00 2026-02-20T21:01:37.018 INFO:teuthology.orchestra.run.trial199.stdout:(26/40): lua-devel-5.4.4-4.el9.x86_64.rpm 69 kB/s | 22 kB 00:00 2026-02-20T21:01:37.027 INFO:teuthology.orchestra.run.trial199.stdout:(27/40): python3-backports-tarfile-1.2.0-1.el9. 550 kB/s | 60 kB 00:00 2026-02-20T21:01:37.229 INFO:teuthology.orchestra.run.trial196.stdout:(1/5): libxslt-1.1.34-12.el9.x86_64.rpm 3.7 MB/s | 233 kB 00:00 2026-02-20T21:01:37.247 INFO:teuthology.orchestra.run.trial196.stdout:(2/5): socat-1.7.4.1-8.el9.x86_64.rpm 16 MB/s | 303 kB 00:00 2026-02-20T21:01:37.257 INFO:teuthology.orchestra.run.trial196.stdout:(3/5): xmlstarlet-1.6.1-20.el9.x86_64.rpm 6.6 MB/s | 64 kB 00:00 2026-02-20T21:01:37.270 INFO:teuthology.orchestra.run.trial196.stdout:(4/5): libcephsqlite-20.2.0-677.gf7870454.el9.x 1.6 MB/s | 164 kB 00:00 2026-02-20T21:01:37.568 INFO:teuthology.orchestra.run.trial199.stdout:(28/40): python3-cherrypy-18.6.1-2.el9.noarch.r 653 kB/s | 358 kB 00:00 2026-02-20T21:01:37.618 INFO:teuthology.orchestra.run.trial199.stdout:(29/40): python3-cheroot-10.0.1-4.el9.noarch.rp 287 kB/s | 173 kB 00:00 2026-02-20T21:01:37.743 INFO:teuthology.orchestra.run.trial199.stdout:(30/40): python3-jaraco-classes-3.2.1-5.el9.noa 101 kB/s | 18 kB 00:00 2026-02-20T21:01:37.753 INFO:teuthology.orchestra.run.trial199.stdout:(31/40): python3-jaraco-8.2.1-3.el9.noarch.rpm 15 kB/s | 11 kB 00:00 2026-02-20T21:01:37.941 INFO:teuthology.orchestra.run.trial199.stdout:(32/40): python3-jaraco-collections-3.0.0-8.el9 72 kB/s | 23 kB 00:00 2026-02-20T21:01:37.953 INFO:teuthology.orchestra.run.trial199.stdout:(33/40): python3-jaraco-context-6.0.1-3.el9.noa 94 kB/s | 20 kB 00:00 2026-02-20T21:01:38.093 INFO:teuthology.orchestra.run.trial199.stdout:(34/40): python3-jaraco-functools-3.5.0-2.el9.n 57 kB/s | 19 kB 00:00 2026-02-20T21:01:38.134 INFO:teuthology.orchestra.run.trial199.stdout:(35/40): python3-more-itertools-8.12.0-2.el9.no 435 kB/s | 79 kB 00:00 2026-02-20T21:01:38.228 INFO:teuthology.orchestra.run.trial199.stdout:(36/40): python3-pyOpenSSL-21.0.0-1.el9.noarch. 964 kB/s | 90 kB 00:00 2026-02-20T21:01:38.261 INFO:teuthology.orchestra.run.trial199.stdout:(37/40): python3-jaraco-text-4.0.0-2.el9.noarch 83 kB/s | 26 kB 00:00 2026-02-20T21:01:38.272 INFO:teuthology.orchestra.run.trial199.stdout:(38/40): python3-tempora-5.0.0-2.el9.noarch.rpm 810 kB/s | 36 kB 00:00 2026-02-20T21:01:38.632 INFO:teuthology.orchestra.run.trial199.stdout:(39/40): python3-portend-3.1.0-2.el9.noarch.rpm 30 kB/s | 16 kB 00:00 2026-02-20T21:01:38.729 INFO:teuthology.orchestra.run.trial199.stdout:(40/40): python3-zc-lockfile-2.0-10.el9.noarch. 43 kB/s | 20 kB 00:00 2026-02-20T21:01:38.732 INFO:teuthology.orchestra.run.trial199.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:01:38.732 INFO:teuthology.orchestra.run.trial199.stdout:Total 7.8 MB/s | 30 MB 00:03 2026-02-20T21:01:38.874 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction check 2026-02-20T21:01:38.893 INFO:teuthology.orchestra.run.trial199.stdout:Transaction check succeeded. 2026-02-20T21:01:38.893 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction test 2026-02-20T21:01:39.037 INFO:teuthology.orchestra.run.trial199.stdout:Transaction test succeeded. 2026-02-20T21:01:39.037 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction 2026-02-20T21:01:39.228 INFO:teuthology.orchestra.run.trial199.stdout: Preparing : 1/1 2026-02-20T21:01:39.230 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/40 2026-02-20T21:01:39.235 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/40 2026-02-20T21:01:39.240 INFO:teuthology.orchestra.run.trial199.stdout: Installing : lua-5.4.4-4.el9.x86_64 3/40 2026-02-20T21:01:39.242 INFO:teuthology.orchestra.run.trial199.stdout: Installing : unzip-6.0-59.el9.x86_64 4/40 2026-02-20T21:01:39.247 INFO:teuthology.orchestra.run.trial199.stdout: Installing : ledmon-libs-1.1.0-3.el9.x86_64 5/40 2026-02-20T21:01:39.272 INFO:teuthology.orchestra.run.trial199.stdout: Installing : zip-3.0-35.el9.x86_64 6/40 2026-02-20T21:01:39.275 INFO:teuthology.orchestra.run.trial199.stdout: Installing : luarocks-3.9.2-5.el9.noarch 7/40 2026-02-20T21:01:39.280 INFO:teuthology.orchestra.run.trial199.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 8/40 2026-02-20T21:01:39.285 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 9/40 2026-02-20T21:01:39.291 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 10/40 2026-02-20T21:01:39.295 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-backports-tarfile-1.2.0-1.el9.noarch 11/40 2026-02-20T21:01:39.300 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-jaraco-context-6.0.1-3.el9.noarch 12/40 2026-02-20T21:01:39.306 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-autocommand-2.2.2-8.el9.noarch 13/40 2026-02-20T21:01:39.309 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 14/40 2026-02-20T21:01:39.315 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 15/40 2026-02-20T21:01:39.318 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-jaraco-text-4.0.0-2.el9.noarch 16/40 2026-02-20T21:01:39.325 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 17/40 2026-02-20T21:01:39.328 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 18/40 2026-02-20T21:01:39.351 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 19/40 2026-02-20T21:01:39.363 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-urllib3-1.26.5-7.el9.noarch 20/40 2026-02-20T21:01:39.369 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-requests-2.25.1-10.el9.noarch 21/40 2026-02-20T21:01:39.378 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-pyparsing-2.4.7-9.el9.noarch 22/40 2026-02-20T21:01:39.385 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-packaging-20.9-5.el9.noarch 23/40 2026-02-20T21:01:39.398 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-ply-3.11-14.el9.noarch 24/40 2026-02-20T21:01:39.413 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 25/40 2026-02-20T21:01:39.487 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 26/40 2026-02-20T21:01:39.496 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-cryptography-36.0.1-5.el9.x86_64 27/40 2026-02-20T21:01:39.519 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 28/40 2026-02-20T21:01:39.548 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-cheroot-10.0.1-4.el9.noarch 29/40 2026-02-20T21:01:39.553 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 30/40 2026-02-20T21:01:39.600 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 31/40 2026-02-20T21:01:39.643 INFO:teuthology.orchestra.run.trial199.stdout: Installing : ceph-mgr-modules-core-2:20.2.0-677.gf7870454.el9.n 32/40 2026-02-20T21:01:39.644 INFO:teuthology.orchestra.run.trial199.stdout: Installing : ceph-mgr-2:20.2.0-677.gf7870454.el9.x86_64 33/40 2026-02-20T21:01:39.665 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: ceph-mgr-2:20.2.0-677.gf7870454.el9.x86_64 33/40 2026-02-20T21:01:39.665 INFO:teuthology.orchestra.run.trial199.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:01:39.665 INFO:teuthology.orchestra.run.trial199.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-02-20T21:01:39.665 INFO:teuthology.orchestra.run.trial199.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-20T21:01:39.665 INFO:teuthology.orchestra.run.trial199.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-20T21:01:39.665 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:39.670 INFO:teuthology.orchestra.run.trial199.stdout: Installing : libconfig-1.7.2-9.el9.x86_64 34/40 2026-02-20T21:01:39.686 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 35/40 2026-02-20T21:01:39.687 INFO:teuthology.orchestra.run.trial199.stdout:Creating group 'libstoragemgmt' with GID 991. 2026-02-20T21:01:39.687 INFO:teuthology.orchestra.run.trial199.stdout:Creating user 'libstoragemgmt' (daemon account for libstoragemgmt) with UID 991 and GID 991. 2026-02-20T21:01:39.687 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:39.696 INFO:teuthology.orchestra.run.trial199.stdout: Installing : libstoragemgmt-1.10.1-1.el9.x86_64 35/40 2026-02-20T21:01:39.719 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 35/40 2026-02-20T21:01:39.719 INFO:teuthology.orchestra.run.trial199.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/libstoragemgmt.service → /usr/lib/systemd/system/libstoragemgmt.service. 2026-02-20T21:01:39.719 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:40.401 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 36/40 2026-02-20T21:01:40.402 INFO:teuthology.orchestra.run.trial199.stdout: Installing : ceph-osd-2:20.2.0-677.gf7870454.el9.x86_64 37/40 2026-02-20T21:01:40.426 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: ceph-osd-2:20.2.0-677.gf7870454.el9.x86_64 37/40 2026-02-20T21:01:40.426 INFO:teuthology.orchestra.run.trial199.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:01:40.427 INFO:teuthology.orchestra.run.trial199.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-02-20T21:01:40.427 INFO:teuthology.orchestra.run.trial199.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-20T21:01:40.427 INFO:teuthology.orchestra.run.trial199.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-20T21:01:40.427 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:40.639 INFO:teuthology.orchestra.run.trial199.stdout: Installing : ceph-mon-2:20.2.0-677.gf7870454.el9.x86_64 38/40 2026-02-20T21:01:40.660 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: ceph-mon-2:20.2.0-677.gf7870454.el9.x86_64 38/40 2026-02-20T21:01:40.660 INFO:teuthology.orchestra.run.trial199.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:01:40.660 INFO:teuthology.orchestra.run.trial199.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-02-20T21:01:40.660 INFO:teuthology.orchestra.run.trial199.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-20T21:01:40.660 INFO:teuthology.orchestra.run.trial199.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-20T21:01:40.660 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:40.757 INFO:teuthology.orchestra.run.trial199.stdout: Installing : ceph-mds-2:20.2.0-677.gf7870454.el9.x86_64 39/40 2026-02-20T21:01:40.779 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: ceph-mds-2:20.2.0-677.gf7870454.el9.x86_64 39/40 2026-02-20T21:01:40.779 INFO:teuthology.orchestra.run.trial199.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:01:40.779 INFO:teuthology.orchestra.run.trial199.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-02-20T21:01:40.779 INFO:teuthology.orchestra.run.trial199.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-20T21:01:40.779 INFO:teuthology.orchestra.run.trial199.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-20T21:01:40.779 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:40.795 INFO:teuthology.orchestra.run.trial199.stdout: Installing : ceph-2:20.2.0-677.gf7870454.el9.x86_64 40/40 2026-02-20T21:01:41.137 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: ceph-2:20.2.0-677.gf7870454.el9.x86_64 40/40 2026-02-20T21:01:41.137 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : ceph-2:20.2.0-677.gf7870454.el9.x86_64 1/40 2026-02-20T21:01:41.137 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : ceph-mds-2:20.2.0-677.gf7870454.el9.x86_64 2/40 2026-02-20T21:01:41.138 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : ceph-mgr-2:20.2.0-677.gf7870454.el9.x86_64 3/40 2026-02-20T21:01:41.138 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : ceph-mon-2:20.2.0-677.gf7870454.el9.x86_64 4/40 2026-02-20T21:01:41.138 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : ceph-osd-2:20.2.0-677.gf7870454.el9.x86_64 5/40 2026-02-20T21:01:41.138 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : ceph-mgr-modules-core-2:20.2.0-677.gf7870454.el9.n 6/40 2026-02-20T21:01:41.138 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 7/40 2026-02-20T21:01:41.138 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 8/40 2026-02-20T21:01:41.138 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 9/40 2026-02-20T21:01:41.138 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 10/40 2026-02-20T21:01:41.138 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-ply-3.11-14.el9.noarch 11/40 2026-02-20T21:01:41.138 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 12/40 2026-02-20T21:01:41.138 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 13/40 2026-02-20T21:01:41.138 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 14/40 2026-02-20T21:01:41.139 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 15/40 2026-02-20T21:01:41.139 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : unzip-6.0-59.el9.x86_64 16/40 2026-02-20T21:01:41.139 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : zip-3.0-35.el9.x86_64 17/40 2026-02-20T21:01:41.139 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 18/40 2026-02-20T21:01:41.139 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : lua-5.4.4-4.el9.x86_64 19/40 2026-02-20T21:01:41.139 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 20/40 2026-02-20T21:01:41.139 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 21/40 2026-02-20T21:01:41.139 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 22/40 2026-02-20T21:01:41.139 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 23/40 2026-02-20T21:01:41.139 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 24/40 2026-02-20T21:01:41.139 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 25/40 2026-02-20T21:01:41.139 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 26/40 2026-02-20T21:01:41.140 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 27/40 2026-02-20T21:01:41.140 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 28/40 2026-02-20T21:01:41.140 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 29/40 2026-02-20T21:01:41.140 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 30/40 2026-02-20T21:01:41.140 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 31/40 2026-02-20T21:01:41.140 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 32/40 2026-02-20T21:01:41.140 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 33/40 2026-02-20T21:01:41.140 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 34/40 2026-02-20T21:01:41.140 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 35/40 2026-02-20T21:01:41.140 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 36/40 2026-02-20T21:01:41.140 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 37/40 2026-02-20T21:01:41.140 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 38/40 2026-02-20T21:01:41.140 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 39/40 2026-02-20T21:01:41.193 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 40/40 2026-02-20T21:01:41.193 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:41.193 INFO:teuthology.orchestra.run.trial199.stdout:Installed: 2026-02-20T21:01:41.193 INFO:teuthology.orchestra.run.trial199.stdout: ceph-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:41.193 INFO:teuthology.orchestra.run.trial199.stdout: ceph-mds-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:41.194 INFO:teuthology.orchestra.run.trial199.stdout: ceph-mgr-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:41.194 INFO:teuthology.orchestra.run.trial199.stdout: ceph-mgr-modules-core-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:01:41.194 INFO:teuthology.orchestra.run.trial199.stdout: ceph-mon-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:41.194 INFO:teuthology.orchestra.run.trial199.stdout: ceph-osd-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:41.194 INFO:teuthology.orchestra.run.trial199.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-02-20T21:01:41.194 INFO:teuthology.orchestra.run.trial199.stdout: libconfig-1.7.2-9.el9.x86_64 2026-02-20T21:01:41.194 INFO:teuthology.orchestra.run.trial199.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-20T21:01:41.194 INFO:teuthology.orchestra.run.trial199.stdout: lua-5.4.4-4.el9.x86_64 2026-02-20T21:01:41.194 INFO:teuthology.orchestra.run.trial199.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-02-20T21:01:41.194 INFO:teuthology.orchestra.run.trial199.stdout: luarocks-3.9.2-5.el9.noarch 2026-02-20T21:01:41.195 INFO:teuthology.orchestra.run.trial199.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-02-20T21:01:41.195 INFO:teuthology.orchestra.run.trial199.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-02-20T21:01:41.195 INFO:teuthology.orchestra.run.trial199.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-02-20T21:01:41.195 INFO:teuthology.orchestra.run.trial199.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-02-20T21:01:41.195 INFO:teuthology.orchestra.run.trial199.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-02-20T21:01:41.195 INFO:teuthology.orchestra.run.trial199.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-02-20T21:01:41.195 INFO:teuthology.orchestra.run.trial199.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-02-20T21:01:41.195 INFO:teuthology.orchestra.run.trial199.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-02-20T21:01:41.195 INFO:teuthology.orchestra.run.trial199.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-02-20T21:01:41.195 INFO:teuthology.orchestra.run.trial199.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-02-20T21:01:41.196 INFO:teuthology.orchestra.run.trial199.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-02-20T21:01:41.196 INFO:teuthology.orchestra.run.trial199.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-02-20T21:01:41.196 INFO:teuthology.orchestra.run.trial199.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-02-20T21:01:41.196 INFO:teuthology.orchestra.run.trial199.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-20T21:01:41.196 INFO:teuthology.orchestra.run.trial199.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-02-20T21:01:41.196 INFO:teuthology.orchestra.run.trial199.stdout: python3-packaging-20.9-5.el9.noarch 2026-02-20T21:01:41.196 INFO:teuthology.orchestra.run.trial199.stdout: python3-ply-3.11-14.el9.noarch 2026-02-20T21:01:41.196 INFO:teuthology.orchestra.run.trial199.stdout: python3-portend-3.1.0-2.el9.noarch 2026-02-20T21:01:41.196 INFO:teuthology.orchestra.run.trial199.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-02-20T21:01:41.196 INFO:teuthology.orchestra.run.trial199.stdout: python3-pycparser-2.20-6.el9.noarch 2026-02-20T21:01:41.196 INFO:teuthology.orchestra.run.trial199.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-02-20T21:01:41.197 INFO:teuthology.orchestra.run.trial199.stdout: python3-requests-2.25.1-10.el9.noarch 2026-02-20T21:01:41.197 INFO:teuthology.orchestra.run.trial199.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-02-20T21:01:41.197 INFO:teuthology.orchestra.run.trial199.stdout: python3-toml-0.10.2-6.el9.noarch 2026-02-20T21:01:41.197 INFO:teuthology.orchestra.run.trial199.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-02-20T21:01:41.197 INFO:teuthology.orchestra.run.trial199.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-02-20T21:01:41.197 INFO:teuthology.orchestra.run.trial199.stdout: unzip-6.0-59.el9.x86_64 2026-02-20T21:01:41.197 INFO:teuthology.orchestra.run.trial199.stdout: zip-3.0-35.el9.x86_64 2026-02-20T21:01:41.197 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:41.197 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:01:41.229 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install ceph-base 2026-02-20T21:01:41.538 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:26 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:01:41.605 INFO:teuthology.orchestra.run.trial199.stdout:Package ceph-base-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:01:41.622 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:01:41.623 INFO:teuthology.orchestra.run.trial199.stdout:Nothing to do. 2026-02-20T21:01:41.623 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:01:41.645 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install cephadm 2026-02-20T21:01:41.948 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:26 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:01:42.031 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:01:42.032 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:42.032 INFO:teuthology.orchestra.run.trial199.stdout: Package Arch Version Repository Size 2026-02-20T21:01:42.032 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:42.032 INFO:teuthology.orchestra.run.trial199.stdout:Installing: 2026-02-20T21:01:42.032 INFO:teuthology.orchestra.run.trial199.stdout: cephadm noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 1.0 M 2026-02-20T21:01:42.032 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:42.032 INFO:teuthology.orchestra.run.trial199.stdout:Transaction Summary 2026-02-20T21:01:42.032 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:42.033 INFO:teuthology.orchestra.run.trial199.stdout:Install 1 Package 2026-02-20T21:01:42.033 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:42.033 INFO:teuthology.orchestra.run.trial199.stdout:Total download size: 1.0 M 2026-02-20T21:01:42.033 INFO:teuthology.orchestra.run.trial199.stdout:Installed size: 1.0 M 2026-02-20T21:01:42.033 INFO:teuthology.orchestra.run.trial199.stdout:Downloading Packages: 2026-02-20T21:01:42.672 INFO:teuthology.orchestra.run.trial199.stdout:cephadm-20.2.0-677.gf7870454.el9.noarch.rpm 1.6 MB/s | 1.0 MB 00:00 2026-02-20T21:01:42.673 INFO:teuthology.orchestra.run.trial199.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:01:42.673 INFO:teuthology.orchestra.run.trial199.stdout:Total 1.6 MB/s | 1.0 MB 00:00 2026-02-20T21:01:42.673 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction check 2026-02-20T21:01:42.675 INFO:teuthology.orchestra.run.trial199.stdout:Transaction check succeeded. 2026-02-20T21:01:42.675 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction test 2026-02-20T21:01:42.679 INFO:teuthology.orchestra.run.trial199.stdout:Transaction test succeeded. 2026-02-20T21:01:42.679 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction 2026-02-20T21:01:42.697 INFO:teuthology.orchestra.run.trial199.stdout: Preparing : 1/1 2026-02-20T21:01:42.755 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: cephadm-2:20.2.0-677.gf7870454.el9.noarch 1/1 2026-02-20T21:01:42.769 INFO:teuthology.orchestra.run.trial199.stdout: Installing : cephadm-2:20.2.0-677.gf7870454.el9.noarch 1/1 2026-02-20T21:01:42.831 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: cephadm-2:20.2.0-677.gf7870454.el9.noarch 1/1 2026-02-20T21:01:42.886 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : cephadm-2:20.2.0-677.gf7870454.el9.noarch 1/1 2026-02-20T21:01:42.886 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:42.886 INFO:teuthology.orchestra.run.trial199.stdout:Installed: 2026-02-20T21:01:42.886 INFO:teuthology.orchestra.run.trial199.stdout: cephadm-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:01:42.886 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:42.886 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:01:42.912 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install ceph-immutable-object-cache 2026-02-20T21:01:43.230 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:28 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:01:43.315 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:01:43.315 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:43.315 INFO:teuthology.orchestra.run.trial199.stdout: Package Arch Version Repo Size 2026-02-20T21:01:43.315 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:43.315 INFO:teuthology.orchestra.run.trial199.stdout:Installing: 2026-02-20T21:01:43.315 INFO:teuthology.orchestra.run.trial199.stdout: ceph-immutable-object-cache x86_64 2:20.2.0-677.gf7870454.el9 ceph 154 k 2026-02-20T21:01:43.315 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:43.316 INFO:teuthology.orchestra.run.trial199.stdout:Transaction Summary 2026-02-20T21:01:43.316 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:43.316 INFO:teuthology.orchestra.run.trial199.stdout:Install 1 Package 2026-02-20T21:01:43.316 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:43.316 INFO:teuthology.orchestra.run.trial199.stdout:Total download size: 154 k 2026-02-20T21:01:43.316 INFO:teuthology.orchestra.run.trial199.stdout:Installed size: 447 k 2026-02-20T21:01:43.316 INFO:teuthology.orchestra.run.trial199.stdout:Downloading Packages: 2026-02-20T21:01:43.555 INFO:teuthology.orchestra.run.trial199.stdout:ceph-immutable-object-cache-20.2.0-677.gf787045 643 kB/s | 154 kB 00:00 2026-02-20T21:01:43.555 INFO:teuthology.orchestra.run.trial199.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:01:43.555 INFO:teuthology.orchestra.run.trial199.stdout:Total 640 kB/s | 154 kB 00:00 2026-02-20T21:01:43.555 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction check 2026-02-20T21:01:43.560 INFO:teuthology.orchestra.run.trial199.stdout:Transaction check succeeded. 2026-02-20T21:01:43.560 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction test 2026-02-20T21:01:43.586 INFO:teuthology.orchestra.run.trial199.stdout:Transaction test succeeded. 2026-02-20T21:01:43.586 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction 2026-02-20T21:01:43.635 INFO:teuthology.orchestra.run.trial199.stdout: Preparing : 1/1 2026-02-20T21:01:43.636 INFO:teuthology.orchestra.run.trial199.stdout: Installing : ceph-immutable-object-cache-2:20.2.0-677.gf7870454.e 1/1 2026-02-20T21:01:43.658 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: ceph-immutable-object-cache-2:20.2.0-677.gf7870454.e 1/1 2026-02-20T21:01:43.658 INFO:teuthology.orchestra.run.trial199.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:01:43.658 INFO:teuthology.orchestra.run.trial199.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-02-20T21:01:43.658 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:43.958 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : ceph-immutable-object-cache-2:20.2.0-677.gf7870454.e 1/1 2026-02-20T21:01:43.958 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:43.958 INFO:teuthology.orchestra.run.trial199.stdout:Installed: 2026-02-20T21:01:43.958 INFO:teuthology.orchestra.run.trial199.stdout: ceph-immutable-object-cache-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:43.959 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:43.959 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:01:43.985 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install ceph-mgr 2026-02-20T21:01:44.292 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:29 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:01:44.358 INFO:teuthology.orchestra.run.trial199.stdout:Package ceph-mgr-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:01:44.375 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:01:44.376 INFO:teuthology.orchestra.run.trial199.stdout:Nothing to do. 2026-02-20T21:01:44.377 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:01:44.397 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install ceph-mgr-dashboard 2026-02-20T21:01:44.707 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:29 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:01:44.789 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:01:44.790 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:44.790 INFO:teuthology.orchestra.run.trial199.stdout: Package Arch Version Repository Size 2026-02-20T21:01:44.791 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:44.791 INFO:teuthology.orchestra.run.trial199.stdout:Installing: 2026-02-20T21:01:44.791 INFO:teuthology.orchestra.run.trial199.stdout: ceph-mgr-dashboard noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 11 M 2026-02-20T21:01:44.791 INFO:teuthology.orchestra.run.trial199.stdout:Installing dependencies: 2026-02-20T21:01:44.791 INFO:teuthology.orchestra.run.trial199.stdout: abseil-cpp x86_64 20211102.0-4.el9 epel 551 k 2026-02-20T21:01:44.791 INFO:teuthology.orchestra.run.trial199.stdout: ceph-grafana-dashboards noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 43 k 2026-02-20T21:01:44.791 INFO:teuthology.orchestra.run.trial199.stdout: ceph-prometheus-alerts noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 17 k 2026-02-20T21:01:44.791 INFO:teuthology.orchestra.run.trial199.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2026-02-20T21:01:44.791 INFO:teuthology.orchestra.run.trial199.stdout: protobuf x86_64 3.14.0-17.el9 appstream 1.0 M 2026-02-20T21:01:44.791 INFO:teuthology.orchestra.run.trial199.stdout: protobuf-compiler x86_64 3.14.0-17.el9 crb 862 k 2026-02-20T21:01:44.792 INFO:teuthology.orchestra.run.trial199.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2026-02-20T21:01:44.792 INFO:teuthology.orchestra.run.trial199.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2026-02-20T21:01:44.792 INFO:teuthology.orchestra.run.trial199.stdout: python3-jmespath noarch 1.0.1-1.el9 appstream 48 k 2026-02-20T21:01:44.792 INFO:teuthology.orchestra.run.trial199.stdout: python3-protobuf noarch 3.14.0-17.el9 appstream 267 k 2026-02-20T21:01:44.792 INFO:teuthology.orchestra.run.trial199.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2026-02-20T21:01:44.792 INFO:teuthology.orchestra.run.trial199.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2026-02-20T21:01:44.792 INFO:teuthology.orchestra.run.trial199.stdout: python3-xmltodict noarch 0.12.0-15.el9 epel 22 k 2026-02-20T21:01:44.792 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:44.792 INFO:teuthology.orchestra.run.trial199.stdout:Transaction Summary 2026-02-20T21:01:44.793 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:44.793 INFO:teuthology.orchestra.run.trial199.stdout:Install 14 Packages 2026-02-20T21:01:44.793 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:44.793 INFO:teuthology.orchestra.run.trial199.stdout:Total download size: 16 M 2026-02-20T21:01:44.793 INFO:teuthology.orchestra.run.trial199.stdout:Installed size: 158 M 2026-02-20T21:01:44.793 INFO:teuthology.orchestra.run.trial199.stdout:Downloading Packages: 2026-02-20T21:01:45.144 INFO:teuthology.orchestra.run.trial199.stdout:(1/14): ceph-prometheus-alerts-20.2.0-677.gf787 157 kB/s | 17 kB 00:00 2026-02-20T21:01:45.153 INFO:teuthology.orchestra.run.trial199.stdout:(2/14): ceph-grafana-dashboards-20.2.0-677.gf78 360 kB/s | 43 kB 00:00 2026-02-20T21:01:45.696 INFO:teuthology.orchestra.run.trial199.stdout:(3/14): python3-jmespath-1.0.1-1.el9.noarch.rpm 88 kB/s | 48 kB 00:00 2026-02-20T21:01:45.959 INFO:teuthology.orchestra.run.trial196.stdout:(5/5): ceph-test-20.2.0-677.gf7870454.el9.x86_6 9.5 MB/s | 84 MB 00:08 2026-02-20T21:01:45.960 INFO:teuthology.orchestra.run.trial196.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:01:45.960 INFO:teuthology.orchestra.run.trial196.stdout:Total 9.3 MB/s | 85 MB 00:09 2026-02-20T21:01:45.986 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction check 2026-02-20T21:01:45.995 INFO:teuthology.orchestra.run.trial196.stdout:Transaction check succeeded. 2026-02-20T21:01:45.996 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction test 2026-02-20T21:01:46.283 INFO:teuthology.orchestra.run.trial196.stdout:Transaction test succeeded. 2026-02-20T21:01:46.283 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction 2026-02-20T21:01:46.597 INFO:teuthology.orchestra.run.trial196.stdout: Preparing : 1/1 2026-02-20T21:01:46.602 INFO:teuthology.orchestra.run.trial196.stdout: Installing : socat-1.7.4.1-8.el9.x86_64 1/5 2026-02-20T21:01:46.608 INFO:teuthology.orchestra.run.trial196.stdout: Installing : libxslt-1.1.34-12.el9.x86_64 2/5 2026-02-20T21:01:46.616 INFO:teuthology.orchestra.run.trial196.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 3/5 2026-02-20T21:01:46.616 INFO:teuthology.orchestra.run.trial196.stdout: Installing : libcephsqlite-2:20.2.0-677.gf7870454.el9.x86_64 4/5 2026-02-20T21:01:46.781 INFO:teuthology.orchestra.run.trial199.stdout:(4/14): protobuf-3.14.0-17.el9.x86_64.rpm 628 kB/s | 1.0 MB 00:01 2026-02-20T21:01:46.818 INFO:teuthology.orchestra.run.trial199.stdout:(5/14): python3-protobuf-3.14.0-17.el9.noarch.r 238 kB/s | 267 kB 00:01 2026-02-20T21:01:46.847 INFO:teuthology.orchestra.run.trial199.stdout:(6/14): protobuf-compiler-3.14.0-17.el9.x86_64. 13 MB/s | 862 kB 00:00 2026-02-20T21:01:46.991 INFO:teuthology.orchestra.run.trial199.stdout:(7/14): grpc-data-1.46.7-10.el9.noarch.rpm 135 kB/s | 19 kB 00:00 2026-02-20T21:01:47.152 INFO:teuthology.orchestra.run.trial199.stdout:(8/14): abseil-cpp-20211102.0-4.el9.x86_64.rpm 1.6 MB/s | 551 kB 00:00 2026-02-20T21:01:47.195 INFO:teuthology.orchestra.run.trial199.stdout:(9/14): python3-grpcio-tools-1.46.7-10.el9.x86_ 3.3 MB/s | 144 kB 00:00 2026-02-20T21:01:47.231 INFO:teuthology.orchestra.run.trial199.stdout:(10/14): python3-grpcio-1.46.7-10.el9.x86_64.rp 8.5 MB/s | 2.0 MB 00:00 2026-02-20T21:01:47.236 INFO:teuthology.orchestra.run.trial199.stdout:(11/14): python3-repoze-lru-0.7-16.el9.noarch.r 750 kB/s | 31 kB 00:00 2026-02-20T21:01:47.273 INFO:teuthology.orchestra.run.trial199.stdout:(12/14): python3-routes-2.5.1-5.el9.noarch.rpm 4.4 MB/s | 188 kB 00:00 2026-02-20T21:01:47.277 INFO:teuthology.orchestra.run.trial199.stdout:(13/14): python3-xmltodict-0.12.0-15.el9.noarch 539 kB/s | 22 kB 00:00 2026-02-20T21:01:48.105 INFO:teuthology.orchestra.run.trial199.stdout:(14/14): ceph-mgr-dashboard-20.2.0-677.gf787045 3.4 MB/s | 11 MB 00:03 2026-02-20T21:01:48.107 INFO:teuthology.orchestra.run.trial199.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:01:48.107 INFO:teuthology.orchestra.run.trial199.stdout:Total 4.8 MB/s | 16 MB 00:03 2026-02-20T21:01:48.143 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction check 2026-02-20T21:01:48.153 INFO:teuthology.orchestra.run.trial199.stdout:Transaction check succeeded. 2026-02-20T21:01:48.153 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction test 2026-02-20T21:01:48.248 INFO:teuthology.orchestra.run.trial199.stdout:Transaction test succeeded. 2026-02-20T21:01:48.249 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction 2026-02-20T21:01:48.366 INFO:teuthology.orchestra.run.trial199.stdout: Preparing : 1/1 2026-02-20T21:01:48.375 INFO:teuthology.orchestra.run.trial199.stdout: Installing : protobuf-3.14.0-17.el9.x86_64 1/14 2026-02-20T21:01:48.378 INFO:teuthology.orchestra.run.trial199.stdout: Installing : protobuf-compiler-3.14.0-17.el9.x86_64 2/14 2026-02-20T21:01:48.383 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-xmltodict-0.12.0-15.el9.noarch 3/14 2026-02-20T21:01:48.395 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 4/14 2026-02-20T21:01:48.397 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 5/14 2026-02-20T21:01:48.424 INFO:teuthology.orchestra.run.trial199.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 6/14 2026-02-20T21:01:48.466 INFO:teuthology.orchestra.run.trial199.stdout: Installing : abseil-cpp-20211102.0-4.el9.x86_64 7/14 2026-02-20T21:01:48.495 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 8/14 2026-02-20T21:01:48.505 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-protobuf-3.14.0-17.el9.noarch 9/14 2026-02-20T21:01:48.512 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 10/14 2026-02-20T21:01:48.514 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-jmespath-1.0.1-1.el9.noarch 11/14 2026-02-20T21:01:48.521 INFO:teuthology.orchestra.run.trial199.stdout: Installing : ceph-prometheus-alerts-2:20.2.0-677.gf7870454.el9. 12/14 2026-02-20T21:01:49.646 INFO:teuthology.orchestra.run.trial199.stdout: Installing : ceph-grafana-dashboards-2:20.2.0-677.gf7870454.el9 13/14 2026-02-20T21:01:49.651 INFO:teuthology.orchestra.run.trial199.stdout: Installing : ceph-mgr-dashboard-2:20.2.0-677.gf7870454.el9.noar 14/14 2026-02-20T21:01:49.862 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: ceph-mgr-dashboard-2:20.2.0-677.gf7870454.el9.noar 14/14 2026-02-20T21:01:49.862 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : ceph-grafana-dashboards-2:20.2.0-677.gf7870454.el9 1/14 2026-02-20T21:01:49.862 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : ceph-mgr-dashboard-2:20.2.0-677.gf7870454.el9.noar 2/14 2026-02-20T21:01:49.862 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : ceph-prometheus-alerts-2:20.2.0-677.gf7870454.el9. 3/14 2026-02-20T21:01:49.862 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 4/14 2026-02-20T21:01:49.862 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-jmespath-1.0.1-1.el9.noarch 5/14 2026-02-20T21:01:49.862 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 6/14 2026-02-20T21:01:49.862 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 7/14 2026-02-20T21:01:49.863 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 8/14 2026-02-20T21:01:49.863 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 9/14 2026-02-20T21:01:49.863 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 10/14 2026-02-20T21:01:49.863 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 11/14 2026-02-20T21:01:49.863 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 12/14 2026-02-20T21:01:49.863 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 13/14 2026-02-20T21:01:49.916 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-xmltodict-0.12.0-15.el9.noarch 14/14 2026-02-20T21:01:49.916 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:49.916 INFO:teuthology.orchestra.run.trial199.stdout:Installed: 2026-02-20T21:01:49.916 INFO:teuthology.orchestra.run.trial199.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-02-20T21:01:49.917 INFO:teuthology.orchestra.run.trial199.stdout: ceph-grafana-dashboards-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:01:49.917 INFO:teuthology.orchestra.run.trial199.stdout: ceph-mgr-dashboard-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:01:49.917 INFO:teuthology.orchestra.run.trial199.stdout: ceph-prometheus-alerts-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:01:49.917 INFO:teuthology.orchestra.run.trial199.stdout: grpc-data-1.46.7-10.el9.noarch 2026-02-20T21:01:49.917 INFO:teuthology.orchestra.run.trial199.stdout: protobuf-3.14.0-17.el9.x86_64 2026-02-20T21:01:49.917 INFO:teuthology.orchestra.run.trial199.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-02-20T21:01:49.917 INFO:teuthology.orchestra.run.trial199.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-02-20T21:01:49.917 INFO:teuthology.orchestra.run.trial199.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-02-20T21:01:49.917 INFO:teuthology.orchestra.run.trial199.stdout: python3-jmespath-1.0.1-1.el9.noarch 2026-02-20T21:01:49.917 INFO:teuthology.orchestra.run.trial199.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-02-20T21:01:49.917 INFO:teuthology.orchestra.run.trial199.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-02-20T21:01:49.918 INFO:teuthology.orchestra.run.trial199.stdout: python3-routes-2.5.1-5.el9.noarch 2026-02-20T21:01:49.918 INFO:teuthology.orchestra.run.trial199.stdout: python3-xmltodict-0.12.0-15.el9.noarch 2026-02-20T21:01:49.918 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:49.918 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:01:49.950 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install ceph-mgr-diskprediction-local 2026-02-20T21:01:50.253 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:35 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:01:50.326 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: libcephsqlite-2:20.2.0-677.gf7870454.el9.x86_64 4/5 2026-02-20T21:01:50.335 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:01:50.336 INFO:teuthology.orchestra.run.trial199.stdout:====================================================================================== 2026-02-20T21:01:50.336 INFO:teuthology.orchestra.run.trial199.stdout: Package Arch Version Repository Size 2026-02-20T21:01:50.336 INFO:teuthology.orchestra.run.trial199.stdout:====================================================================================== 2026-02-20T21:01:50.336 INFO:teuthology.orchestra.run.trial199.stdout:Installing: 2026-02-20T21:01:50.336 INFO:teuthology.orchestra.run.trial199.stdout: ceph-mgr-diskprediction-local noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 7.4 M 2026-02-20T21:01:50.337 INFO:teuthology.orchestra.run.trial199.stdout:Installing dependencies: 2026-02-20T21:01:50.337 INFO:teuthology.orchestra.run.trial199.stdout: flexiblas x86_64 3.0.4-9.el9 appstream 30 k 2026-02-20T21:01:50.337 INFO:teuthology.orchestra.run.trial199.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 appstream 3.0 M 2026-02-20T21:01:50.337 INFO:teuthology.orchestra.run.trial199.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 appstream 15 k 2026-02-20T21:01:50.337 INFO:teuthology.orchestra.run.trial199.stdout: libgfortran x86_64 11.5.0-14.el9 baseos 794 k 2026-02-20T21:01:50.337 INFO:teuthology.orchestra.run.trial199.stdout: libquadmath x86_64 11.5.0-14.el9 baseos 184 k 2026-02-20T21:01:50.337 INFO:teuthology.orchestra.run.trial199.stdout: openblas x86_64 0.3.29-1.el9 appstream 42 k 2026-02-20T21:01:50.337 INFO:teuthology.orchestra.run.trial199.stdout: openblas-openmp x86_64 0.3.29-1.el9 appstream 5.3 M 2026-02-20T21:01:50.337 INFO:teuthology.orchestra.run.trial199.stdout: python3-devel x86_64 3.9.25-3.el9 appstream 244 k 2026-02-20T21:01:50.338 INFO:teuthology.orchestra.run.trial199.stdout: python3-numpy x86_64 1:1.23.5-2.el9 appstream 6.1 M 2026-02-20T21:01:50.338 INFO:teuthology.orchestra.run.trial199.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 appstream 442 k 2026-02-20T21:01:50.338 INFO:teuthology.orchestra.run.trial199.stdout: python3-scipy x86_64 1.9.3-2.el9 appstream 19 M 2026-02-20T21:01:50.338 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:50.338 INFO:teuthology.orchestra.run.trial199.stdout:Transaction Summary 2026-02-20T21:01:50.338 INFO:teuthology.orchestra.run.trial199.stdout:====================================================================================== 2026-02-20T21:01:50.338 INFO:teuthology.orchestra.run.trial199.stdout:Install 12 Packages 2026-02-20T21:01:50.338 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:50.339 INFO:teuthology.orchestra.run.trial199.stdout:Total download size: 43 M 2026-02-20T21:01:50.339 INFO:teuthology.orchestra.run.trial199.stdout:Installed size: 234 M 2026-02-20T21:01:50.339 INFO:teuthology.orchestra.run.trial199.stdout:Downloading Packages: 2026-02-20T21:01:50.345 INFO:teuthology.orchestra.run.trial196.stdout: Installing : ceph-test-2:20.2.0-677.gf7870454.el9.x86_64 5/5 2026-02-20T21:01:50.681 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: ceph-test-2:20.2.0-677.gf7870454.el9.x86_64 5/5 2026-02-20T21:01:50.681 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : ceph-test-2:20.2.0-677.gf7870454.el9.x86_64 1/5 2026-02-20T21:01:50.681 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : libcephsqlite-2:20.2.0-677.gf7870454.el9.x86_64 2/5 2026-02-20T21:01:50.681 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 3/5 2026-02-20T21:01:50.682 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 4/5 2026-02-20T21:01:50.734 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/5 2026-02-20T21:01:50.734 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:50.734 INFO:teuthology.orchestra.run.trial196.stdout:Installed: 2026-02-20T21:01:50.734 INFO:teuthology.orchestra.run.trial196.stdout: ceph-test-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:50.734 INFO:teuthology.orchestra.run.trial196.stdout: libcephsqlite-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:50.734 INFO:teuthology.orchestra.run.trial196.stdout: libxslt-1.1.34-12.el9.x86_64 2026-02-20T21:01:50.734 INFO:teuthology.orchestra.run.trial196.stdout: socat-1.7.4.1-8.el9.x86_64 2026-02-20T21:01:50.734 INFO:teuthology.orchestra.run.trial196.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2026-02-20T21:01:50.735 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:50.735 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:01:50.770 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install ceph 2026-02-20T21:01:51.079 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:28 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:01:51.163 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:01:51.165 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:01:51.165 INFO:teuthology.orchestra.run.trial196.stdout: Package Arch Version Repository Size 2026-02-20T21:01:51.165 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:01:51.165 INFO:teuthology.orchestra.run.trial196.stdout:Installing: 2026-02-20T21:01:51.165 INFO:teuthology.orchestra.run.trial196.stdout: ceph x86_64 2:20.2.0-677.gf7870454.el9 ceph 6.5 k 2026-02-20T21:01:51.165 INFO:teuthology.orchestra.run.trial196.stdout:Installing dependencies: 2026-02-20T21:01:51.165 INFO:teuthology.orchestra.run.trial196.stdout: ceph-mds x86_64 2:20.2.0-677.gf7870454.el9 ceph 2.3 M 2026-02-20T21:01:51.165 INFO:teuthology.orchestra.run.trial196.stdout: ceph-mgr x86_64 2:20.2.0-677.gf7870454.el9 ceph 961 k 2026-02-20T21:01:51.166 INFO:teuthology.orchestra.run.trial196.stdout: ceph-mgr-modules-core noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 290 k 2026-02-20T21:01:51.166 INFO:teuthology.orchestra.run.trial196.stdout: ceph-mon x86_64 2:20.2.0-677.gf7870454.el9 ceph 5.0 M 2026-02-20T21:01:51.166 INFO:teuthology.orchestra.run.trial196.stdout: ceph-osd x86_64 2:20.2.0-677.gf7870454.el9 ceph 17 M 2026-02-20T21:01:51.166 INFO:teuthology.orchestra.run.trial196.stdout: ledmon-libs x86_64 1.1.0-3.el9 baseos 40 k 2026-02-20T21:01:51.166 INFO:teuthology.orchestra.run.trial196.stdout: libconfig x86_64 1.7.2-9.el9 baseos 72 k 2026-02-20T21:01:51.166 INFO:teuthology.orchestra.run.trial196.stdout: libstoragemgmt x86_64 1.10.1-1.el9 appstream 246 k 2026-02-20T21:01:51.166 INFO:teuthology.orchestra.run.trial196.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2026-02-20T21:01:51.166 INFO:teuthology.orchestra.run.trial196.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2026-02-20T21:01:51.166 INFO:teuthology.orchestra.run.trial196.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2026-02-20T21:01:51.166 INFO:teuthology.orchestra.run.trial196.stdout: python3-autocommand noarch 2.2.2-8.el9 epel 29 k 2026-02-20T21:01:51.166 INFO:teuthology.orchestra.run.trial196.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 epel 60 k 2026-02-20T21:01:51.166 INFO:teuthology.orchestra.run.trial196.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2026-02-20T21:01:51.166 INFO:teuthology.orchestra.run.trial196.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2026-02-20T21:01:51.167 INFO:teuthology.orchestra.run.trial196.stdout: python3-cheroot noarch 10.0.1-4.el9 epel 173 k 2026-02-20T21:01:51.167 INFO:teuthology.orchestra.run.trial196.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2026-02-20T21:01:51.167 INFO:teuthology.orchestra.run.trial196.stdout: python3-cryptography x86_64 36.0.1-5.el9 baseos 1.2 M 2026-02-20T21:01:51.167 INFO:teuthology.orchestra.run.trial196.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2026-02-20T21:01:51.167 INFO:teuthology.orchestra.run.trial196.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2026-02-20T21:01:51.167 INFO:teuthology.orchestra.run.trial196.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2026-02-20T21:01:51.167 INFO:teuthology.orchestra.run.trial196.stdout: python3-jaraco-context noarch 6.0.1-3.el9 epel 20 k 2026-02-20T21:01:51.167 INFO:teuthology.orchestra.run.trial196.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2026-02-20T21:01:51.167 INFO:teuthology.orchestra.run.trial196.stdout: python3-jaraco-text noarch 4.0.0-2.el9 epel 26 k 2026-02-20T21:01:51.167 INFO:teuthology.orchestra.run.trial196.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 appstream 177 k 2026-02-20T21:01:51.167 INFO:teuthology.orchestra.run.trial196.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2026-02-20T21:01:51.167 INFO:teuthology.orchestra.run.trial196.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2026-02-20T21:01:51.168 INFO:teuthology.orchestra.run.trial196.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2026-02-20T21:01:51.168 INFO:teuthology.orchestra.run.trial196.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2026-02-20T21:01:51.168 INFO:teuthology.orchestra.run.trial196.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2026-02-20T21:01:51.168 INFO:teuthology.orchestra.run.trial196.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2026-02-20T21:01:51.168 INFO:teuthology.orchestra.run.trial196.stdout: python3-pyparsing noarch 2.4.7-9.el9 baseos 150 k 2026-02-20T21:01:51.168 INFO:teuthology.orchestra.run.trial196.stdout: python3-requests noarch 2.25.1-10.el9 baseos 126 k 2026-02-20T21:01:51.168 INFO:teuthology.orchestra.run.trial196.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2026-02-20T21:01:51.168 INFO:teuthology.orchestra.run.trial196.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2026-02-20T21:01:51.168 INFO:teuthology.orchestra.run.trial196.stdout: python3-urllib3 noarch 1.26.5-7.el9 baseos 218 k 2026-02-20T21:01:51.168 INFO:teuthology.orchestra.run.trial196.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2026-02-20T21:01:51.168 INFO:teuthology.orchestra.run.trial196.stdout: unzip x86_64 6.0-59.el9 baseos 182 k 2026-02-20T21:01:51.168 INFO:teuthology.orchestra.run.trial196.stdout: zip x86_64 3.0-35.el9 baseos 266 k 2026-02-20T21:01:51.168 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:51.169 INFO:teuthology.orchestra.run.trial196.stdout:Transaction Summary 2026-02-20T21:01:51.169 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:01:51.169 INFO:teuthology.orchestra.run.trial196.stdout:Install 40 Packages 2026-02-20T21:01:51.169 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:51.170 INFO:teuthology.orchestra.run.trial196.stdout:Total download size: 30 M 2026-02-20T21:01:51.170 INFO:teuthology.orchestra.run.trial196.stdout:Installed size: 100 M 2026-02-20T21:01:51.170 INFO:teuthology.orchestra.run.trial196.stdout:Downloading Packages: 2026-02-20T21:01:51.307 INFO:teuthology.orchestra.run.trial199.stdout:(1/12): libquadmath-11.5.0-14.el9.x86_64.rpm 233 kB/s | 184 kB 00:00 2026-02-20T21:01:51.359 INFO:teuthology.orchestra.run.trial199.stdout:(2/12): ceph-mgr-diskprediction-local-20.2.0-67 8.8 MB/s | 7.4 MB 00:00 2026-02-20T21:01:51.359 INFO:teuthology.orchestra.run.trial199.stdout:(3/12): flexiblas-3.0.4-9.el9.x86_64.rpm 568 kB/s | 30 kB 00:00 2026-02-20T21:01:51.380 INFO:teuthology.orchestra.run.trial199.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-9.el9.x 695 kB/s | 15 kB 00:00 2026-02-20T21:01:51.402 INFO:teuthology.orchestra.run.trial199.stdout:(5/12): openblas-0.3.29-1.el9.x86_64.rpm 1.9 MB/s | 42 kB 00:00 2026-02-20T21:01:51.528 INFO:teuthology.orchestra.run.trial199.stdout:(6/12): flexiblas-netlib-3.0.4-9.el9.x86_64.rpm 18 MB/s | 3.0 MB 00:00 2026-02-20T21:01:51.536 INFO:teuthology.orchestra.run.trial199.stdout:(7/12): openblas-openmp-0.3.29-1.el9.x86_64.rpm 40 MB/s | 5.3 MB 00:00 2026-02-20T21:01:51.554 INFO:teuthology.orchestra.run.trial199.stdout:(8/12): python3-devel-3.9.25-3.el9.x86_64.rpm 9.3 MB/s | 244 kB 00:00 2026-02-20T21:01:51.579 INFO:teuthology.orchestra.run.trial199.stdout:(9/12): python3-numpy-f2py-1.23.5-2.el9.x86_64. 17 MB/s | 442 kB 00:00 2026-02-20T21:01:51.592 INFO:teuthology.orchestra.run.trial199.stdout:(10/12): python3-numpy-1.23.5-2.el9.x86_64.rpm 109 MB/s | 6.1 MB 00:00 2026-02-20T21:01:51.711 INFO:teuthology.orchestra.run.trial196.stdout:(1/40): ceph-20.2.0-677.gf7870454.el9.x86_64.rp 129 kB/s | 6.5 kB 00:00 2026-02-20T21:01:51.732 INFO:teuthology.orchestra.run.trial199.stdout:(11/12): python3-scipy-1.9.3-2.el9.x86_64.rpm 126 MB/s | 19 MB 00:00 2026-02-20T21:01:51.881 INFO:teuthology.orchestra.run.trial196.stdout:(2/40): ceph-mgr-20.2.0-677.gf7870454.el9.x86_6 4.3 MB/s | 961 kB 00:00 2026-02-20T21:01:52.002 INFO:teuthology.orchestra.run.trial199.stdout:(12/12): libgfortran-11.5.0-14.el9.x86_64.rpm 535 kB/s | 794 kB 00:01 2026-02-20T21:01:52.006 INFO:teuthology.orchestra.run.trial199.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:01:52.006 INFO:teuthology.orchestra.run.trial199.stdout:Total 26 MB/s | 43 MB 00:01 2026-02-20T21:01:52.022 INFO:teuthology.orchestra.run.trial196.stdout:(3/40): ceph-mds-20.2.0-677.gf7870454.el9.x86_6 6.5 MB/s | 2.3 MB 00:00 2026-02-20T21:01:52.121 INFO:teuthology.orchestra.run.trial196.stdout:(4/40): ceph-mgr-modules-core-20.2.0-677.gf7870 2.9 MB/s | 290 kB 00:00 2026-02-20T21:01:52.164 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction check 2026-02-20T21:01:52.178 INFO:teuthology.orchestra.run.trial199.stdout:Transaction check succeeded. 2026-02-20T21:01:52.178 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction test 2026-02-20T21:01:52.379 INFO:teuthology.orchestra.run.trial199.stdout:Transaction test succeeded. 2026-02-20T21:01:52.379 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction 2026-02-20T21:01:52.405 INFO:teuthology.orchestra.run.trial196.stdout:(5/40): ledmon-libs-1.1.0-3.el9.x86_64.rpm 143 kB/s | 40 kB 00:00 2026-02-20T21:01:52.545 INFO:teuthology.orchestra.run.trial196.stdout:(6/40): ceph-mon-20.2.0-677.gf7870454.el9.x86_6 6.1 MB/s | 5.0 MB 00:00 2026-02-20T21:01:52.595 INFO:teuthology.orchestra.run.trial199.stdout: Preparing : 1/1 2026-02-20T21:01:52.595 INFO:teuthology.orchestra.run.trial196.stdout:(7/40): libconfig-1.7.2-9.el9.x86_64.rpm 378 kB/s | 72 kB 00:00 2026-02-20T21:01:52.598 INFO:teuthology.orchestra.run.trial199.stdout: Installing : flexiblas-3.0.4-9.el9.x86_64 1/12 2026-02-20T21:01:52.604 INFO:teuthology.orchestra.run.trial199.stdout: Installing : libquadmath-11.5.0-14.el9.x86_64 2/12 2026-02-20T21:01:52.638 INFO:teuthology.orchestra.run.trial199.stdout: Installing : libgfortran-11.5.0-14.el9.x86_64 3/12 2026-02-20T21:01:52.642 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-devel-3.9.25-3.el9.x86_64 4/12 2026-02-20T21:01:52.701 INFO:teuthology.orchestra.run.trial199.stdout: Installing : openblas-0.3.29-1.el9.x86_64 5/12 2026-02-20T21:01:52.704 INFO:teuthology.orchestra.run.trial199.stdout: Installing : openblas-openmp-0.3.29-1.el9.x86_64 6/12 2026-02-20T21:01:52.725 INFO:teuthology.orchestra.run.trial199.stdout: Installing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 7/12 2026-02-20T21:01:52.986 INFO:teuthology.orchestra.run.trial196.stdout:(8/40): python3-cryptography-36.0.1-5.el9.x86_6 3.2 MB/s | 1.2 MB 00:00 2026-02-20T21:01:53.044 INFO:teuthology.orchestra.run.trial199.stdout: Installing : flexiblas-netlib-3.0.4-9.el9.x86_64 8/12 2026-02-20T21:01:53.049 INFO:teuthology.orchestra.run.trial196.stdout:(9/40): python3-cffi-1.14.5-5.el9.x86_64.rpm 502 kB/s | 253 kB 00:00 2026-02-20T21:01:53.068 INFO:teuthology.orchestra.run.trial196.stdout:(10/40): python3-ply-3.11-14.el9.noarch.rpm 1.3 MB/s | 106 kB 00:00 2026-02-20T21:01:53.117 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-numpy-1:1.23.5-2.el9.x86_64 9/12 2026-02-20T21:01:53.151 INFO:teuthology.orchestra.run.trial196.stdout:(11/40): python3-pyparsing-2.4.7-9.el9.noarch.r 1.8 MB/s | 150 kB 00:00 2026-02-20T21:01:53.157 INFO:teuthology.orchestra.run.trial196.stdout:(12/40): python3-pycparser-2.20-6.el9.noarch.rp 1.2 MB/s | 135 kB 00:00 2026-02-20T21:01:53.234 INFO:teuthology.orchestra.run.trial196.stdout:(13/40): python3-requests-2.25.1-10.el9.noarch. 1.5 MB/s | 126 kB 00:00 2026-02-20T21:01:53.295 INFO:teuthology.orchestra.run.trial196.stdout:(14/40): ceph-osd-20.2.0-677.gf7870454.el9.x86_ 12 MB/s | 17 MB 00:01 2026-02-20T21:01:53.295 INFO:teuthology.orchestra.run.trial196.stdout:(15/40): python3-urllib3-1.26.5-7.el9.noarch.rp 1.5 MB/s | 218 kB 00:00 2026-02-20T21:01:53.318 INFO:teuthology.orchestra.run.trial196.stdout:(16/40): unzip-6.0-59.el9.x86_64.rpm 2.1 MB/s | 182 kB 00:00 2026-02-20T21:01:53.781 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 10/12 2026-02-20T21:01:53.807 INFO:teuthology.orchestra.run.trial196.stdout:(17/40): zip-3.0-35.el9.x86_64.rpm 519 kB/s | 266 kB 00:00 2026-02-20T21:01:53.994 INFO:teuthology.orchestra.run.trial196.stdout:(18/40): lua-5.4.4-4.el9.x86_64.rpm 278 kB/s | 188 kB 00:00 2026-02-20T21:01:54.048 INFO:teuthology.orchestra.run.trial196.stdout:(19/40): libstoragemgmt-1.10.1-1.el9.x86_64.rpm 327 kB/s | 246 kB 00:00 2026-02-20T21:01:54.148 INFO:teuthology.orchestra.run.trial196.stdout:(20/40): python3-toml-0.10.2-6.el9.noarch.rpm 418 kB/s | 42 kB 00:00 2026-02-20T21:01:54.172 INFO:teuthology.orchestra.run.trial196.stdout:(21/40): python3-packaging-20.9-5.el9.noarch.rp 433 kB/s | 77 kB 00:00 2026-02-20T21:01:54.229 INFO:teuthology.orchestra.run.trial196.stdout:(22/40): lua-devel-5.4.4-4.el9.x86_64.rpm 274 kB/s | 22 kB 00:00 2026-02-20T21:01:54.233 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-scipy-1.9.3-2.el9.x86_64 11/12 2026-02-20T21:01:54.234 INFO:teuthology.orchestra.run.trial199.stdout: Installing : ceph-mgr-diskprediction-local-2:20.2.0-677.gf78704 12/12 2026-02-20T21:01:54.338 INFO:teuthology.orchestra.run.trial196.stdout:(23/40): python3-autocommand-2.2.2-8.el9.noarch 272 kB/s | 29 kB 00:00 2026-02-20T21:01:54.371 INFO:teuthology.orchestra.run.trial196.stdout:(24/40): luarocks-3.9.2-5.el9.noarch.rpm 764 kB/s | 151 kB 00:00 2026-02-20T21:01:54.410 INFO:teuthology.orchestra.run.trial196.stdout:(25/40): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 1.1 MB/s | 43 kB 00:00 2026-02-20T21:01:54.412 INFO:teuthology.orchestra.run.trial196.stdout:(26/40): python3-backports-tarfile-1.2.0-1.el9. 805 kB/s | 60 kB 00:00 2026-02-20T21:01:54.465 INFO:teuthology.orchestra.run.trial196.stdout:(27/40): python3-libstoragemgmt-1.10.1-1.el9.x8 269 kB/s | 177 kB 00:00 2026-02-20T21:01:54.470 INFO:teuthology.orchestra.run.trial196.stdout:(28/40): python3-cheroot-10.0.1-4.el9.noarch.rp 2.8 MB/s | 173 kB 00:00 2026-02-20T21:01:54.503 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:20.2.0-677.gf78704 12/12 2026-02-20T21:01:54.504 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : ceph-mgr-diskprediction-local-2:20.2.0-677.gf78704 1/12 2026-02-20T21:01:54.504 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 2/12 2026-02-20T21:01:54.504 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 3/12 2026-02-20T21:01:54.504 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 4/12 2026-02-20T21:01:54.504 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 5/12 2026-02-20T21:01:54.504 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 6/12 2026-02-20T21:01:54.504 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 7/12 2026-02-20T21:01:54.504 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 8/12 2026-02-20T21:01:54.504 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-devel-3.9.25-3.el9.x86_64 9/12 2026-02-20T21:01:54.504 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 10/12 2026-02-20T21:01:54.504 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 11/12 2026-02-20T21:01:54.504 INFO:teuthology.orchestra.run.trial196.stdout:(29/40): python3-jaraco-classes-3.2.1-5.el9.noa 512 kB/s | 18 kB 00:00 2026-02-20T21:01:54.526 INFO:teuthology.orchestra.run.trial196.stdout:(30/40): python3-cherrypy-18.6.1-2.el9.noarch.r 3.1 MB/s | 358 kB 00:00 2026-02-20T21:01:54.530 INFO:teuthology.orchestra.run.trial196.stdout:(31/40): python3-jaraco-8.2.1-3.el9.noarch.rpm 163 kB/s | 11 kB 00:00 2026-02-20T21:01:54.539 INFO:teuthology.orchestra.run.trial196.stdout:(32/40): python3-jaraco-collections-3.0.0-8.el9 659 kB/s | 23 kB 00:00 2026-02-20T21:01:54.560 INFO:teuthology.orchestra.run.trial196.stdout:(33/40): python3-jaraco-context-6.0.1-3.el9.noa 578 kB/s | 20 kB 00:00 2026-02-20T21:01:54.562 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 12/12 2026-02-20T21:01:54.562 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:54.563 INFO:teuthology.orchestra.run.trial199.stdout:Installed: 2026-02-20T21:01:54.563 INFO:teuthology.orchestra.run.trial199.stdout: ceph-mgr-diskprediction-local-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:01:54.563 INFO:teuthology.orchestra.run.trial199.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-02-20T21:01:54.563 INFO:teuthology.orchestra.run.trial199.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-02-20T21:01:54.563 INFO:teuthology.orchestra.run.trial199.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-02-20T21:01:54.563 INFO:teuthology.orchestra.run.trial199.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-02-20T21:01:54.563 INFO:teuthology.orchestra.run.trial199.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-02-20T21:01:54.563 INFO:teuthology.orchestra.run.trial199.stdout: openblas-0.3.29-1.el9.x86_64 2026-02-20T21:01:54.563 INFO:teuthology.orchestra.run.trial199.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-02-20T21:01:54.563 INFO:teuthology.orchestra.run.trial199.stdout: python3-devel-3.9.25-3.el9.x86_64 2026-02-20T21:01:54.563 INFO:teuthology.orchestra.run.trial199.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-02-20T21:01:54.563 INFO:teuthology.orchestra.run.trial199.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-02-20T21:01:54.563 INFO:teuthology.orchestra.run.trial199.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-02-20T21:01:54.563 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:54.563 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:01:54.575 INFO:teuthology.orchestra.run.trial196.stdout:(34/40): python3-jaraco-text-4.0.0-2.el9.noarch 753 kB/s | 26 kB 00:00 2026-02-20T21:01:54.576 INFO:teuthology.orchestra.run.trial196.stdout:(35/40): python3-jaraco-functools-3.5.0-2.el9.n 429 kB/s | 19 kB 00:00 2026-02-20T21:01:54.597 INFO:teuthology.orchestra.run.trial196.stdout:(36/40): python3-more-itertools-8.12.0-2.el9.no 2.1 MB/s | 79 kB 00:00 2026-02-20T21:01:54.600 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install ceph-mgr-rook 2026-02-20T21:01:54.609 INFO:teuthology.orchestra.run.trial196.stdout:(37/40): python3-portend-3.1.0-2.el9.noarch.rpm 479 kB/s | 16 kB 00:00 2026-02-20T21:01:54.631 INFO:teuthology.orchestra.run.trial196.stdout:(38/40): python3-tempora-5.0.0-2.el9.noarch.rpm 1.0 MB/s | 36 kB 00:00 2026-02-20T21:01:54.644 INFO:teuthology.orchestra.run.trial196.stdout:(39/40): python3-zc-lockfile-2.0-10.el9.noarch. 581 kB/s | 20 kB 00:00 2026-02-20T21:01:54.671 INFO:teuthology.orchestra.run.trial196.stdout:(40/40): python3-pyOpenSSL-21.0.0-1.el9.noarch. 940 kB/s | 90 kB 00:00 2026-02-20T21:01:54.672 INFO:teuthology.orchestra.run.trial196.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:01:54.672 INFO:teuthology.orchestra.run.trial196.stdout:Total 8.6 MB/s | 30 MB 00:03 2026-02-20T21:01:54.813 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction check 2026-02-20T21:01:54.832 INFO:teuthology.orchestra.run.trial196.stdout:Transaction check succeeded. 2026-02-20T21:01:54.832 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction test 2026-02-20T21:01:54.906 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:39 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:01:54.977 INFO:teuthology.orchestra.run.trial196.stdout:Transaction test succeeded. 2026-02-20T21:01:54.977 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction 2026-02-20T21:01:54.989 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:01:54.989 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:54.990 INFO:teuthology.orchestra.run.trial199.stdout: Package Arch Version Repository Size 2026-02-20T21:01:54.990 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:54.990 INFO:teuthology.orchestra.run.trial199.stdout:Installing: 2026-02-20T21:01:54.990 INFO:teuthology.orchestra.run.trial199.stdout: ceph-mgr-rook noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 50 k 2026-02-20T21:01:54.990 INFO:teuthology.orchestra.run.trial199.stdout:Installing dependencies: 2026-02-20T21:01:54.990 INFO:teuthology.orchestra.run.trial199.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2026-02-20T21:01:54.990 INFO:teuthology.orchestra.run.trial199.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2026-02-20T21:01:54.990 INFO:teuthology.orchestra.run.trial199.stdout: python3-google-auth noarch 1:2.45.0-1.el9 epel 254 k 2026-02-20T21:01:54.990 INFO:teuthology.orchestra.run.trial199.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 epel 1.0 M 2026-02-20T21:01:54.991 INFO:teuthology.orchestra.run.trial199.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2026-02-20T21:01:54.991 INFO:teuthology.orchestra.run.trial199.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2026-02-20T21:01:54.991 INFO:teuthology.orchestra.run.trial199.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2026-02-20T21:01:54.991 INFO:teuthology.orchestra.run.trial199.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2026-02-20T21:01:54.991 INFO:teuthology.orchestra.run.trial199.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2026-02-20T21:01:54.991 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:54.991 INFO:teuthology.orchestra.run.trial199.stdout:Transaction Summary 2026-02-20T21:01:54.991 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:54.991 INFO:teuthology.orchestra.run.trial199.stdout:Install 10 Packages 2026-02-20T21:01:54.991 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:54.992 INFO:teuthology.orchestra.run.trial199.stdout:Total download size: 2.0 M 2026-02-20T21:01:54.992 INFO:teuthology.orchestra.run.trial199.stdout:Installed size: 25 M 2026-02-20T21:01:54.992 INFO:teuthology.orchestra.run.trial199.stdout:Downloading Packages: 2026-02-20T21:01:55.168 INFO:teuthology.orchestra.run.trial196.stdout: Preparing : 1/1 2026-02-20T21:01:55.170 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/40 2026-02-20T21:01:55.174 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/40 2026-02-20T21:01:55.179 INFO:teuthology.orchestra.run.trial196.stdout: Installing : lua-5.4.4-4.el9.x86_64 3/40 2026-02-20T21:01:55.181 INFO:teuthology.orchestra.run.trial196.stdout: Installing : unzip-6.0-59.el9.x86_64 4/40 2026-02-20T21:01:55.187 INFO:teuthology.orchestra.run.trial196.stdout: Installing : ledmon-libs-1.1.0-3.el9.x86_64 5/40 2026-02-20T21:01:55.211 INFO:teuthology.orchestra.run.trial196.stdout: Installing : zip-3.0-35.el9.x86_64 6/40 2026-02-20T21:01:55.215 INFO:teuthology.orchestra.run.trial196.stdout: Installing : luarocks-3.9.2-5.el9.noarch 7/40 2026-02-20T21:01:55.220 INFO:teuthology.orchestra.run.trial196.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 8/40 2026-02-20T21:01:55.225 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 9/40 2026-02-20T21:01:55.231 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 10/40 2026-02-20T21:01:55.235 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-backports-tarfile-1.2.0-1.el9.noarch 11/40 2026-02-20T21:01:55.241 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-jaraco-context-6.0.1-3.el9.noarch 12/40 2026-02-20T21:01:55.246 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-autocommand-2.2.2-8.el9.noarch 13/40 2026-02-20T21:01:55.250 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 14/40 2026-02-20T21:01:55.255 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 15/40 2026-02-20T21:01:55.259 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-jaraco-text-4.0.0-2.el9.noarch 16/40 2026-02-20T21:01:55.266 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 17/40 2026-02-20T21:01:55.269 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 18/40 2026-02-20T21:01:55.292 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 19/40 2026-02-20T21:01:55.304 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-urllib3-1.26.5-7.el9.noarch 20/40 2026-02-20T21:01:55.310 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-requests-2.25.1-10.el9.noarch 21/40 2026-02-20T21:01:55.320 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-pyparsing-2.4.7-9.el9.noarch 22/40 2026-02-20T21:01:55.326 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-packaging-20.9-5.el9.noarch 23/40 2026-02-20T21:01:55.339 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-ply-3.11-14.el9.noarch 24/40 2026-02-20T21:01:55.355 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 25/40 2026-02-20T21:01:55.431 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 26/40 2026-02-20T21:01:55.440 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-cryptography-36.0.1-5.el9.x86_64 27/40 2026-02-20T21:01:55.444 INFO:teuthology.orchestra.run.trial199.stdout:(1/10): python3-pyasn1-0.4.8-6.el9.noarch.rpm 3.1 MB/s | 159 kB 00:00 2026-02-20T21:01:55.452 INFO:teuthology.orchestra.run.trial199.stdout:(2/10): python3-pyasn1-modules-0.4.8-6.el9.noar 4.7 MB/s | 279 kB 00:00 2026-02-20T21:01:55.455 INFO:teuthology.orchestra.run.trial199.stdout:(3/10): python3-requests-oauthlib-1.3.0-12.el9. 5.3 MB/s | 54 kB 00:00 2026-02-20T21:01:55.463 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 28/40 2026-02-20T21:01:55.467 INFO:teuthology.orchestra.run.trial199.stdout:(4/10): ceph-mgr-rook-20.2.0-677.gf7870454.el9. 691 kB/s | 50 kB 00:00 2026-02-20T21:01:55.492 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-cheroot-10.0.1-4.el9.noarch 29/40 2026-02-20T21:01:55.498 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 30/40 2026-02-20T21:01:55.526 INFO:teuthology.orchestra.run.trial199.stdout:(5/10): python3-certifi-2023.05.07-4.el9.noarch 196 kB/s | 14 kB 00:00 2026-02-20T21:01:55.542 INFO:teuthology.orchestra.run.trial199.stdout:(6/10): python3-cachetools-4.2.4-1.el9.noarch.r 356 kB/s | 32 kB 00:00 2026-02-20T21:01:55.545 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 31/40 2026-02-20T21:01:55.581 INFO:teuthology.orchestra.run.trial199.stdout:(7/10): python3-rsa-4.9-2.el9.noarch.rpm 1.5 MB/s | 59 kB 00:00 2026-02-20T21:01:55.585 INFO:teuthology.orchestra.run.trial199.stdout:(8/10): python3-google-auth-2.45.0-1.el9.noarch 2.1 MB/s | 254 kB 00:00 2026-02-20T21:01:55.588 INFO:teuthology.orchestra.run.trial196.stdout: Installing : ceph-mgr-modules-core-2:20.2.0-677.gf7870454.el9.n 32/40 2026-02-20T21:01:55.589 INFO:teuthology.orchestra.run.trial196.stdout: Installing : ceph-mgr-2:20.2.0-677.gf7870454.el9.x86_64 33/40 2026-02-20T21:01:55.613 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: ceph-mgr-2:20.2.0-677.gf7870454.el9.x86_64 33/40 2026-02-20T21:01:55.613 INFO:teuthology.orchestra.run.trial196.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:01:55.613 INFO:teuthology.orchestra.run.trial196.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-02-20T21:01:55.613 INFO:teuthology.orchestra.run.trial196.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-20T21:01:55.613 INFO:teuthology.orchestra.run.trial196.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-20T21:01:55.613 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:55.614 INFO:teuthology.orchestra.run.trial199.stdout:(9/10): python3-websocket-client-1.2.3-2.el9.no 2.6 MB/s | 90 kB 00:00 2026-02-20T21:01:55.617 INFO:teuthology.orchestra.run.trial196.stdout: Installing : libconfig-1.7.2-9.el9.x86_64 34/40 2026-02-20T21:01:55.634 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 35/40 2026-02-20T21:01:55.634 INFO:teuthology.orchestra.run.trial196.stdout:Creating group 'libstoragemgmt' with GID 991. 2026-02-20T21:01:55.634 INFO:teuthology.orchestra.run.trial196.stdout:Creating user 'libstoragemgmt' (daemon account for libstoragemgmt) with UID 991 and GID 991. 2026-02-20T21:01:55.634 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:55.642 INFO:teuthology.orchestra.run.trial196.stdout: Installing : libstoragemgmt-1.10.1-1.el9.x86_64 35/40 2026-02-20T21:01:55.651 INFO:teuthology.orchestra.run.trial199.stdout:(10/10): python3-kubernetes-26.1.0-3.el9.noarch 8.2 MB/s | 1.0 MB 00:00 2026-02-20T21:01:55.652 INFO:teuthology.orchestra.run.trial199.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:01:55.652 INFO:teuthology.orchestra.run.trial199.stdout:Total 3.0 MB/s | 2.0 MB 00:00 2026-02-20T21:01:55.666 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 35/40 2026-02-20T21:01:55.666 INFO:teuthology.orchestra.run.trial196.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/libstoragemgmt.service → /usr/lib/systemd/system/libstoragemgmt.service. 2026-02-20T21:01:55.666 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:55.676 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction check 2026-02-20T21:01:55.684 INFO:teuthology.orchestra.run.trial199.stdout:Transaction check succeeded. 2026-02-20T21:01:55.684 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction test 2026-02-20T21:01:55.724 INFO:teuthology.orchestra.run.trial199.stdout:Transaction test succeeded. 2026-02-20T21:01:55.724 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction 2026-02-20T21:01:55.799 INFO:teuthology.orchestra.run.trial199.stdout: Preparing : 1/1 2026-02-20T21:01:55.849 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/10 2026-02-20T21:01:55.862 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/10 2026-02-20T21:01:55.877 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/10 2026-02-20T21:01:55.881 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/10 2026-02-20T21:01:55.889 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/10 2026-02-20T21:01:55.929 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/10 2026-02-20T21:01:55.941 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-google-auth-1:2.45.0-1.el9.noarch 7/10 2026-02-20T21:01:56.264 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 8/10 2026-02-20T21:01:56.279 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-kubernetes-1:26.1.0-3.el9.noarch 9/10 2026-02-20T21:01:56.279 INFO:teuthology.orchestra.run.trial199.stdout: Installing : ceph-mgr-rook-2:20.2.0-677.gf7870454.el9.noarch 10/10 2026-02-20T21:01:56.347 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 36/40 2026-02-20T21:01:56.349 INFO:teuthology.orchestra.run.trial196.stdout: Installing : ceph-osd-2:20.2.0-677.gf7870454.el9.x86_64 37/40 2026-02-20T21:01:56.373 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: ceph-osd-2:20.2.0-677.gf7870454.el9.x86_64 37/40 2026-02-20T21:01:56.373 INFO:teuthology.orchestra.run.trial196.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:01:56.373 INFO:teuthology.orchestra.run.trial196.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-02-20T21:01:56.374 INFO:teuthology.orchestra.run.trial196.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-20T21:01:56.374 INFO:teuthology.orchestra.run.trial196.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-20T21:01:56.374 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:56.381 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: ceph-mgr-rook-2:20.2.0-677.gf7870454.el9.noarch 10/10 2026-02-20T21:01:56.381 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : ceph-mgr-rook-2:20.2.0-677.gf7870454.el9.noarch 1/10 2026-02-20T21:01:56.381 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 2/10 2026-02-20T21:01:56.381 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 3/10 2026-02-20T21:01:56.381 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 4/10 2026-02-20T21:01:56.381 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 5/10 2026-02-20T21:01:56.381 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 6/10 2026-02-20T21:01:56.381 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 7/10 2026-02-20T21:01:56.382 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 8/10 2026-02-20T21:01:56.382 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 9/10 2026-02-20T21:01:56.438 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 10/10 2026-02-20T21:01:56.438 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:56.438 INFO:teuthology.orchestra.run.trial199.stdout:Installed: 2026-02-20T21:01:56.438 INFO:teuthology.orchestra.run.trial199.stdout: ceph-mgr-rook-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:01:56.438 INFO:teuthology.orchestra.run.trial199.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-02-20T21:01:56.438 INFO:teuthology.orchestra.run.trial199.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-02-20T21:01:56.438 INFO:teuthology.orchestra.run.trial199.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-02-20T21:01:56.439 INFO:teuthology.orchestra.run.trial199.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-02-20T21:01:56.439 INFO:teuthology.orchestra.run.trial199.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2026-02-20T21:01:56.439 INFO:teuthology.orchestra.run.trial199.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2026-02-20T21:01:56.439 INFO:teuthology.orchestra.run.trial199.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-02-20T21:01:56.439 INFO:teuthology.orchestra.run.trial199.stdout: python3-rsa-4.9-2.el9.noarch 2026-02-20T21:01:56.439 INFO:teuthology.orchestra.run.trial199.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-02-20T21:01:56.439 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:56.439 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:01:56.467 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install ceph-mgr-cephadm 2026-02-20T21:01:56.587 INFO:teuthology.orchestra.run.trial196.stdout: Installing : ceph-mon-2:20.2.0-677.gf7870454.el9.x86_64 38/40 2026-02-20T21:01:56.608 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: ceph-mon-2:20.2.0-677.gf7870454.el9.x86_64 38/40 2026-02-20T21:01:56.608 INFO:teuthology.orchestra.run.trial196.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:01:56.608 INFO:teuthology.orchestra.run.trial196.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-02-20T21:01:56.608 INFO:teuthology.orchestra.run.trial196.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-20T21:01:56.608 INFO:teuthology.orchestra.run.trial196.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-20T21:01:56.608 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:56.705 INFO:teuthology.orchestra.run.trial196.stdout: Installing : ceph-mds-2:20.2.0-677.gf7870454.el9.x86_64 39/40 2026-02-20T21:01:56.726 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: ceph-mds-2:20.2.0-677.gf7870454.el9.x86_64 39/40 2026-02-20T21:01:56.726 INFO:teuthology.orchestra.run.trial196.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:01:56.727 INFO:teuthology.orchestra.run.trial196.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-02-20T21:01:56.727 INFO:teuthology.orchestra.run.trial196.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-20T21:01:56.727 INFO:teuthology.orchestra.run.trial196.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-20T21:01:56.727 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:56.743 INFO:teuthology.orchestra.run.trial196.stdout: Installing : ceph-2:20.2.0-677.gf7870454.el9.x86_64 40/40 2026-02-20T21:01:56.771 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:41 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:01:56.855 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:01:56.856 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:56.856 INFO:teuthology.orchestra.run.trial199.stdout: Package Arch Version Repository Size 2026-02-20T21:01:56.856 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:56.856 INFO:teuthology.orchestra.run.trial199.stdout:Installing: 2026-02-20T21:01:56.856 INFO:teuthology.orchestra.run.trial199.stdout: ceph-mgr-cephadm noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 173 k 2026-02-20T21:01:56.856 INFO:teuthology.orchestra.run.trial199.stdout:Installing dependencies: 2026-02-20T21:01:56.856 INFO:teuthology.orchestra.run.trial199.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2026-02-20T21:01:56.856 INFO:teuthology.orchestra.run.trial199.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2026-02-20T21:01:56.856 INFO:teuthology.orchestra.run.trial199.stdout: python3-jinja2 noarch 2.11.3-8.el9 appstream 249 k 2026-02-20T21:01:56.856 INFO:teuthology.orchestra.run.trial199.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2026-02-20T21:01:56.856 INFO:teuthology.orchestra.run.trial199.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2026-02-20T21:01:56.857 INFO:teuthology.orchestra.run.trial199.stdout: python3-typing-extensions noarch 4.15.0-1.el9 epel 86 k 2026-02-20T21:01:56.857 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:56.857 INFO:teuthology.orchestra.run.trial199.stdout:Transaction Summary 2026-02-20T21:01:56.857 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:56.857 INFO:teuthology.orchestra.run.trial199.stdout:Install 7 Packages 2026-02-20T21:01:56.857 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:56.857 INFO:teuthology.orchestra.run.trial199.stdout:Total download size: 7.1 M 2026-02-20T21:01:56.857 INFO:teuthology.orchestra.run.trial199.stdout:Installed size: 33 M 2026-02-20T21:01:56.857 INFO:teuthology.orchestra.run.trial199.stdout:Downloading Packages: 2026-02-20T21:01:57.082 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: ceph-2:20.2.0-677.gf7870454.el9.x86_64 40/40 2026-02-20T21:01:57.083 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : ceph-2:20.2.0-677.gf7870454.el9.x86_64 1/40 2026-02-20T21:01:57.083 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : ceph-mds-2:20.2.0-677.gf7870454.el9.x86_64 2/40 2026-02-20T21:01:57.083 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : ceph-mgr-2:20.2.0-677.gf7870454.el9.x86_64 3/40 2026-02-20T21:01:57.083 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : ceph-mon-2:20.2.0-677.gf7870454.el9.x86_64 4/40 2026-02-20T21:01:57.083 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : ceph-osd-2:20.2.0-677.gf7870454.el9.x86_64 5/40 2026-02-20T21:01:57.083 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : ceph-mgr-modules-core-2:20.2.0-677.gf7870454.el9.n 6/40 2026-02-20T21:01:57.083 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 7/40 2026-02-20T21:01:57.083 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 8/40 2026-02-20T21:01:57.084 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 9/40 2026-02-20T21:01:57.084 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 10/40 2026-02-20T21:01:57.084 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-ply-3.11-14.el9.noarch 11/40 2026-02-20T21:01:57.084 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 12/40 2026-02-20T21:01:57.084 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 13/40 2026-02-20T21:01:57.084 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 14/40 2026-02-20T21:01:57.084 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 15/40 2026-02-20T21:01:57.084 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : unzip-6.0-59.el9.x86_64 16/40 2026-02-20T21:01:57.084 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : zip-3.0-35.el9.x86_64 17/40 2026-02-20T21:01:57.084 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 18/40 2026-02-20T21:01:57.084 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : lua-5.4.4-4.el9.x86_64 19/40 2026-02-20T21:01:57.084 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 20/40 2026-02-20T21:01:57.084 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 21/40 2026-02-20T21:01:57.085 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 22/40 2026-02-20T21:01:57.085 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 23/40 2026-02-20T21:01:57.085 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 24/40 2026-02-20T21:01:57.085 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 25/40 2026-02-20T21:01:57.085 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 26/40 2026-02-20T21:01:57.085 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 27/40 2026-02-20T21:01:57.085 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 28/40 2026-02-20T21:01:57.085 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 29/40 2026-02-20T21:01:57.085 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 30/40 2026-02-20T21:01:57.085 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 31/40 2026-02-20T21:01:57.085 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 32/40 2026-02-20T21:01:57.085 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 33/40 2026-02-20T21:01:57.086 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 34/40 2026-02-20T21:01:57.086 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 35/40 2026-02-20T21:01:57.086 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 36/40 2026-02-20T21:01:57.086 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 37/40 2026-02-20T21:01:57.086 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 38/40 2026-02-20T21:01:57.086 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 39/40 2026-02-20T21:01:57.101 INFO:teuthology.orchestra.run.trial199.stdout:(1/7): ceph-mgr-cephadm-20.2.0-677.gf7870454.el 1.7 MB/s | 173 kB 00:00 2026-02-20T21:01:57.139 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 40/40 2026-02-20T21:01:57.139 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:57.139 INFO:teuthology.orchestra.run.trial196.stdout:Installed: 2026-02-20T21:01:57.140 INFO:teuthology.orchestra.run.trial196.stdout: ceph-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:57.140 INFO:teuthology.orchestra.run.trial196.stdout: ceph-mds-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:57.140 INFO:teuthology.orchestra.run.trial196.stdout: ceph-mgr-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:57.140 INFO:teuthology.orchestra.run.trial196.stdout: ceph-mgr-modules-core-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:01:57.140 INFO:teuthology.orchestra.run.trial196.stdout: ceph-mon-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:57.140 INFO:teuthology.orchestra.run.trial196.stdout: ceph-osd-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:57.140 INFO:teuthology.orchestra.run.trial196.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-02-20T21:01:57.140 INFO:teuthology.orchestra.run.trial196.stdout: libconfig-1.7.2-9.el9.x86_64 2026-02-20T21:01:57.140 INFO:teuthology.orchestra.run.trial196.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-20T21:01:57.140 INFO:teuthology.orchestra.run.trial196.stdout: lua-5.4.4-4.el9.x86_64 2026-02-20T21:01:57.140 INFO:teuthology.orchestra.run.trial196.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-02-20T21:01:57.140 INFO:teuthology.orchestra.run.trial196.stdout: luarocks-3.9.2-5.el9.noarch 2026-02-20T21:01:57.141 INFO:teuthology.orchestra.run.trial196.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-02-20T21:01:57.141 INFO:teuthology.orchestra.run.trial196.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-02-20T21:01:57.141 INFO:teuthology.orchestra.run.trial196.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-02-20T21:01:57.141 INFO:teuthology.orchestra.run.trial196.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-02-20T21:01:57.141 INFO:teuthology.orchestra.run.trial196.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-02-20T21:01:57.141 INFO:teuthology.orchestra.run.trial196.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-02-20T21:01:57.141 INFO:teuthology.orchestra.run.trial196.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-02-20T21:01:57.141 INFO:teuthology.orchestra.run.trial196.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-02-20T21:01:57.141 INFO:teuthology.orchestra.run.trial196.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-02-20T21:01:57.141 INFO:teuthology.orchestra.run.trial196.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-02-20T21:01:57.141 INFO:teuthology.orchestra.run.trial196.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-02-20T21:01:57.141 INFO:teuthology.orchestra.run.trial196.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-02-20T21:01:57.142 INFO:teuthology.orchestra.run.trial196.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-02-20T21:01:57.142 INFO:teuthology.orchestra.run.trial196.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-20T21:01:57.142 INFO:teuthology.orchestra.run.trial196.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-02-20T21:01:57.142 INFO:teuthology.orchestra.run.trial196.stdout: python3-packaging-20.9-5.el9.noarch 2026-02-20T21:01:57.142 INFO:teuthology.orchestra.run.trial196.stdout: python3-ply-3.11-14.el9.noarch 2026-02-20T21:01:57.142 INFO:teuthology.orchestra.run.trial196.stdout: python3-portend-3.1.0-2.el9.noarch 2026-02-20T21:01:57.142 INFO:teuthology.orchestra.run.trial196.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-02-20T21:01:57.142 INFO:teuthology.orchestra.run.trial196.stdout: python3-pycparser-2.20-6.el9.noarch 2026-02-20T21:01:57.142 INFO:teuthology.orchestra.run.trial196.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-02-20T21:01:57.142 INFO:teuthology.orchestra.run.trial196.stdout: python3-requests-2.25.1-10.el9.noarch 2026-02-20T21:01:57.142 INFO:teuthology.orchestra.run.trial196.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-02-20T21:01:57.142 INFO:teuthology.orchestra.run.trial196.stdout: python3-toml-0.10.2-6.el9.noarch 2026-02-20T21:01:57.142 INFO:teuthology.orchestra.run.trial196.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-02-20T21:01:57.143 INFO:teuthology.orchestra.run.trial196.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-02-20T21:01:57.143 INFO:teuthology.orchestra.run.trial196.stdout: unzip-6.0-59.el9.x86_64 2026-02-20T21:01:57.143 INFO:teuthology.orchestra.run.trial196.stdout: zip-3.0-35.el9.x86_64 2026-02-20T21:01:57.143 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:57.143 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:01:57.158 INFO:teuthology.orchestra.run.trial199.stdout:(2/7): python3-jinja2-2.11.3-8.el9.noarch.rpm 1.5 MB/s | 249 kB 00:00 2026-02-20T21:01:57.178 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install ceph-base 2026-02-20T21:01:57.186 INFO:teuthology.orchestra.run.trial199.stdout:(3/7): python3-markupsafe-1.1.1-12.el9.x86_64.r 411 kB/s | 35 kB 00:00 2026-02-20T21:01:57.288 INFO:teuthology.orchestra.run.trial199.stdout:(4/7): python3-babel-2.9.1-2.el9.noarch.rpm 21 MB/s | 6.0 MB 00:00 2026-02-20T21:01:57.465 INFO:teuthology.orchestra.run.trial199.stdout:(5/7): python3-natsort-7.1.1-5.el9.noarch.rpm 207 kB/s | 58 kB 00:00 2026-02-20T21:01:57.485 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:34 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:01:57.551 INFO:teuthology.orchestra.run.trial196.stdout:Package ceph-base-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:01:57.568 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:01:57.569 INFO:teuthology.orchestra.run.trial196.stdout:Nothing to do. 2026-02-20T21:01:57.569 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:01:57.591 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install cephadm 2026-02-20T21:01:57.640 INFO:teuthology.orchestra.run.trial199.stdout:(6/7): python3-typing-extensions-4.15.0-1.el9.n 246 kB/s | 86 kB 00:00 2026-02-20T21:01:57.893 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:34 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:01:57.975 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:01:57.975 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:01:57.975 INFO:teuthology.orchestra.run.trial196.stdout: Package Arch Version Repository Size 2026-02-20T21:01:57.975 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:01:57.975 INFO:teuthology.orchestra.run.trial196.stdout:Installing: 2026-02-20T21:01:57.975 INFO:teuthology.orchestra.run.trial196.stdout: cephadm noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 1.0 M 2026-02-20T21:01:57.976 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:57.976 INFO:teuthology.orchestra.run.trial196.stdout:Transaction Summary 2026-02-20T21:01:57.976 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:01:57.976 INFO:teuthology.orchestra.run.trial196.stdout:Install 1 Package 2026-02-20T21:01:57.976 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:57.976 INFO:teuthology.orchestra.run.trial196.stdout:Total download size: 1.0 M 2026-02-20T21:01:57.976 INFO:teuthology.orchestra.run.trial196.stdout:Installed size: 1.0 M 2026-02-20T21:01:57.976 INFO:teuthology.orchestra.run.trial196.stdout:Downloading Packages: 2026-02-20T21:01:58.103 INFO:teuthology.orchestra.run.trial196.stdout:cephadm-20.2.0-677.gf7870454.el9.noarch.rpm 7.9 MB/s | 1.0 MB 00:00 2026-02-20T21:01:58.103 INFO:teuthology.orchestra.run.trial196.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:01:58.103 INFO:teuthology.orchestra.run.trial196.stdout:Total 7.8 MB/s | 1.0 MB 00:00 2026-02-20T21:01:58.103 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction check 2026-02-20T21:01:58.105 INFO:teuthology.orchestra.run.trial196.stdout:Transaction check succeeded. 2026-02-20T21:01:58.105 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction test 2026-02-20T21:01:58.109 INFO:teuthology.orchestra.run.trial196.stdout:Transaction test succeeded. 2026-02-20T21:01:58.109 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction 2026-02-20T21:01:58.127 INFO:teuthology.orchestra.run.trial196.stdout: Preparing : 1/1 2026-02-20T21:01:58.190 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: cephadm-2:20.2.0-677.gf7870454.el9.noarch 1/1 2026-02-20T21:01:58.204 INFO:teuthology.orchestra.run.trial196.stdout: Installing : cephadm-2:20.2.0-677.gf7870454.el9.noarch 1/1 2026-02-20T21:01:58.238 INFO:teuthology.orchestra.run.trial199.stdout:(7/7): python3-asyncssh-2.13.2-5.el9.noarch.rpm 507 kB/s | 548 kB 00:01 2026-02-20T21:01:58.239 INFO:teuthology.orchestra.run.trial199.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:01:58.240 INFO:teuthology.orchestra.run.trial199.stdout:Total 5.1 MB/s | 7.1 MB 00:01 2026-02-20T21:01:58.273 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: cephadm-2:20.2.0-677.gf7870454.el9.noarch 1/1 2026-02-20T21:01:58.278 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction check 2026-02-20T21:01:58.284 INFO:teuthology.orchestra.run.trial199.stdout:Transaction check succeeded. 2026-02-20T21:01:58.284 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction test 2026-02-20T21:01:58.328 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : cephadm-2:20.2.0-677.gf7870454.el9.noarch 1/1 2026-02-20T21:01:58.328 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:58.328 INFO:teuthology.orchestra.run.trial196.stdout:Installed: 2026-02-20T21:01:58.328 INFO:teuthology.orchestra.run.trial196.stdout: cephadm-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:01:58.328 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:58.328 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:01:58.355 INFO:teuthology.orchestra.run.trial199.stdout:Transaction test succeeded. 2026-02-20T21:01:58.355 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction 2026-02-20T21:01:58.356 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install ceph-immutable-object-cache 2026-02-20T21:01:58.445 INFO:teuthology.orchestra.run.trial199.stdout: Preparing : 1/1 2026-02-20T21:01:58.494 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-typing-extensions-4.15.0-1.el9.noarch 1/7 2026-02-20T21:01:58.504 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/7 2026-02-20T21:01:58.509 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/7 2026-02-20T21:01:58.675 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:35 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:01:58.728 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 4/7 2026-02-20T21:01:58.751 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 5/7 2026-02-20T21:01:58.759 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:01:58.760 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:01:58.760 INFO:teuthology.orchestra.run.trial196.stdout: Package Arch Version Repo Size 2026-02-20T21:01:58.760 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:01:58.760 INFO:teuthology.orchestra.run.trial196.stdout:Installing: 2026-02-20T21:01:58.760 INFO:teuthology.orchestra.run.trial196.stdout: ceph-immutable-object-cache x86_64 2:20.2.0-677.gf7870454.el9 ceph 154 k 2026-02-20T21:01:58.760 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:58.760 INFO:teuthology.orchestra.run.trial196.stdout:Transaction Summary 2026-02-20T21:01:58.761 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:01:58.761 INFO:teuthology.orchestra.run.trial196.stdout:Install 1 Package 2026-02-20T21:01:58.761 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:58.761 INFO:teuthology.orchestra.run.trial196.stdout:Total download size: 154 k 2026-02-20T21:01:58.761 INFO:teuthology.orchestra.run.trial196.stdout:Installed size: 447 k 2026-02-20T21:01:58.761 INFO:teuthology.orchestra.run.trial196.stdout:Downloading Packages: 2026-02-20T21:01:58.775 INFO:teuthology.orchestra.run.trial199.stdout: Installing : python3-jinja2-2.11.3-8.el9.noarch 6/7 2026-02-20T21:01:58.776 INFO:teuthology.orchestra.run.trial199.stdout: Installing : ceph-mgr-cephadm-2:20.2.0-677.gf7870454.el9.noarch 7/7 2026-02-20T21:01:58.863 INFO:teuthology.orchestra.run.trial196.stdout:ceph-immutable-object-cache-20.2.0-677.gf787045 1.5 MB/s | 154 kB 00:00 2026-02-20T21:01:58.864 INFO:teuthology.orchestra.run.trial196.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:01:58.864 INFO:teuthology.orchestra.run.trial196.stdout:Total 1.4 MB/s | 154 kB 00:00 2026-02-20T21:01:58.864 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction check 2026-02-20T21:01:58.868 INFO:teuthology.orchestra.run.trial196.stdout:Transaction check succeeded. 2026-02-20T21:01:58.868 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction test 2026-02-20T21:01:58.882 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: ceph-mgr-cephadm-2:20.2.0-677.gf7870454.el9.noarch 7/7 2026-02-20T21:01:58.882 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : ceph-mgr-cephadm-2:20.2.0-677.gf7870454.el9.noarch 1/7 2026-02-20T21:01:58.882 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/7 2026-02-20T21:01:58.882 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 3/7 2026-02-20T21:01:58.882 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 4/7 2026-02-20T21:01:58.882 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 5/7 2026-02-20T21:01:58.882 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 6/7 2026-02-20T21:01:58.895 INFO:teuthology.orchestra.run.trial196.stdout:Transaction test succeeded. 2026-02-20T21:01:58.895 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction 2026-02-20T21:01:58.940 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 7/7 2026-02-20T21:01:58.940 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:58.940 INFO:teuthology.orchestra.run.trial199.stdout:Installed: 2026-02-20T21:01:58.940 INFO:teuthology.orchestra.run.trial199.stdout: ceph-mgr-cephadm-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:01:58.941 INFO:teuthology.orchestra.run.trial199.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-02-20T21:01:58.941 INFO:teuthology.orchestra.run.trial199.stdout: python3-babel-2.9.1-2.el9.noarch 2026-02-20T21:01:58.941 INFO:teuthology.orchestra.run.trial199.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-02-20T21:01:58.941 INFO:teuthology.orchestra.run.trial199.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-02-20T21:01:58.941 INFO:teuthology.orchestra.run.trial199.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-02-20T21:01:58.941 INFO:teuthology.orchestra.run.trial199.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-02-20T21:01:58.941 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:58.941 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:01:58.943 INFO:teuthology.orchestra.run.trial196.stdout: Preparing : 1/1 2026-02-20T21:01:58.945 INFO:teuthology.orchestra.run.trial196.stdout: Installing : ceph-immutable-object-cache-2:20.2.0-677.gf7870454.e 1/1 2026-02-20T21:01:58.968 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: ceph-immutable-object-cache-2:20.2.0-677.gf7870454.e 1/1 2026-02-20T21:01:58.968 INFO:teuthology.orchestra.run.trial196.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:01:58.968 INFO:teuthology.orchestra.run.trial196.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-02-20T21:01:58.968 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:58.970 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install ceph-fuse 2026-02-20T21:01:59.280 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:44 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:01:59.282 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : ceph-immutable-object-cache-2:20.2.0-677.gf7870454.e 1/1 2026-02-20T21:01:59.282 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:59.282 INFO:teuthology.orchestra.run.trial196.stdout:Installed: 2026-02-20T21:01:59.282 INFO:teuthology.orchestra.run.trial196.stdout: ceph-immutable-object-cache-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:59.282 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:01:59.283 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:01:59.311 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install ceph-mgr 2026-02-20T21:01:59.364 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:01:59.364 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:59.364 INFO:teuthology.orchestra.run.trial199.stdout: Package Architecture Version Repository Size 2026-02-20T21:01:59.364 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:59.364 INFO:teuthology.orchestra.run.trial199.stdout:Installing: 2026-02-20T21:01:59.364 INFO:teuthology.orchestra.run.trial199.stdout: ceph-fuse x86_64 2:20.2.0-677.gf7870454.el9 ceph 943 k 2026-02-20T21:01:59.365 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:59.365 INFO:teuthology.orchestra.run.trial199.stdout:Transaction Summary 2026-02-20T21:01:59.365 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:01:59.365 INFO:teuthology.orchestra.run.trial199.stdout:Install 1 Package 2026-02-20T21:01:59.365 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:59.365 INFO:teuthology.orchestra.run.trial199.stdout:Total download size: 943 k 2026-02-20T21:01:59.365 INFO:teuthology.orchestra.run.trial199.stdout:Installed size: 2.7 M 2026-02-20T21:01:59.365 INFO:teuthology.orchestra.run.trial199.stdout:Downloading Packages: 2026-02-20T21:01:59.489 INFO:teuthology.orchestra.run.trial199.stdout:ceph-fuse-20.2.0-677.gf7870454.el9.x86_64.rpm 7.4 MB/s | 943 kB 00:00 2026-02-20T21:01:59.489 INFO:teuthology.orchestra.run.trial199.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:01:59.489 INFO:teuthology.orchestra.run.trial199.stdout:Total 7.3 MB/s | 943 kB 00:00 2026-02-20T21:01:59.489 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction check 2026-02-20T21:01:59.494 INFO:teuthology.orchestra.run.trial199.stdout:Transaction check succeeded. 2026-02-20T21:01:59.494 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction test 2026-02-20T21:01:59.528 INFO:teuthology.orchestra.run.trial199.stdout:Transaction test succeeded. 2026-02-20T21:01:59.528 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction 2026-02-20T21:01:59.616 INFO:teuthology.orchestra.run.trial199.stdout: Preparing : 1/1 2026-02-20T21:01:59.617 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:36 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:01:59.633 INFO:teuthology.orchestra.run.trial199.stdout: Installing : ceph-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:01:59.682 INFO:teuthology.orchestra.run.trial196.stdout:Package ceph-mgr-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:01:59.700 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:01:59.700 INFO:teuthology.orchestra.run.trial196.stdout:Nothing to do. 2026-02-20T21:01:59.701 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:01:59.723 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install ceph-mgr-dashboard 2026-02-20T21:01:59.847 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: ceph-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:01:59.904 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : ceph-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:01:59.904 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:59.905 INFO:teuthology.orchestra.run.trial199.stdout:Installed: 2026-02-20T21:01:59.905 INFO:teuthology.orchestra.run.trial199.stdout: ceph-fuse-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:01:59.905 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:01:59.905 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:01:59.933 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install ceph-volume 2026-02-20T21:02:00.037 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:37 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:02:00.121 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:02:00.122 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:00.122 INFO:teuthology.orchestra.run.trial196.stdout: Package Arch Version Repository Size 2026-02-20T21:02:00.122 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:00.122 INFO:teuthology.orchestra.run.trial196.stdout:Installing: 2026-02-20T21:02:00.122 INFO:teuthology.orchestra.run.trial196.stdout: ceph-mgr-dashboard noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 11 M 2026-02-20T21:02:00.122 INFO:teuthology.orchestra.run.trial196.stdout:Installing dependencies: 2026-02-20T21:02:00.122 INFO:teuthology.orchestra.run.trial196.stdout: abseil-cpp x86_64 20211102.0-4.el9 epel 551 k 2026-02-20T21:02:00.122 INFO:teuthology.orchestra.run.trial196.stdout: ceph-grafana-dashboards noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 43 k 2026-02-20T21:02:00.122 INFO:teuthology.orchestra.run.trial196.stdout: ceph-prometheus-alerts noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 17 k 2026-02-20T21:02:00.122 INFO:teuthology.orchestra.run.trial196.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2026-02-20T21:02:00.122 INFO:teuthology.orchestra.run.trial196.stdout: protobuf x86_64 3.14.0-17.el9 appstream 1.0 M 2026-02-20T21:02:00.122 INFO:teuthology.orchestra.run.trial196.stdout: protobuf-compiler x86_64 3.14.0-17.el9 crb 862 k 2026-02-20T21:02:00.123 INFO:teuthology.orchestra.run.trial196.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2026-02-20T21:02:00.123 INFO:teuthology.orchestra.run.trial196.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2026-02-20T21:02:00.123 INFO:teuthology.orchestra.run.trial196.stdout: python3-jmespath noarch 1.0.1-1.el9 appstream 48 k 2026-02-20T21:02:00.123 INFO:teuthology.orchestra.run.trial196.stdout: python3-protobuf noarch 3.14.0-17.el9 appstream 267 k 2026-02-20T21:02:00.123 INFO:teuthology.orchestra.run.trial196.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2026-02-20T21:02:00.123 INFO:teuthology.orchestra.run.trial196.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2026-02-20T21:02:00.123 INFO:teuthology.orchestra.run.trial196.stdout: python3-xmltodict noarch 0.12.0-15.el9 epel 22 k 2026-02-20T21:02:00.123 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:00.123 INFO:teuthology.orchestra.run.trial196.stdout:Transaction Summary 2026-02-20T21:02:00.123 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:00.123 INFO:teuthology.orchestra.run.trial196.stdout:Install 14 Packages 2026-02-20T21:02:00.123 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:00.123 INFO:teuthology.orchestra.run.trial196.stdout:Total download size: 16 M 2026-02-20T21:02:00.123 INFO:teuthology.orchestra.run.trial196.stdout:Installed size: 158 M 2026-02-20T21:02:00.123 INFO:teuthology.orchestra.run.trial196.stdout:Downloading Packages: 2026-02-20T21:02:00.238 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:45 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:02:00.322 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:02:00.322 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:02:00.322 INFO:teuthology.orchestra.run.trial199.stdout: Package Arch Version Repository Size 2026-02-20T21:02:00.322 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:02:00.322 INFO:teuthology.orchestra.run.trial199.stdout:Installing: 2026-02-20T21:02:00.322 INFO:teuthology.orchestra.run.trial199.stdout: ceph-volume noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 297 k 2026-02-20T21:02:00.322 INFO:teuthology.orchestra.run.trial199.stdout:Installing dependencies: 2026-02-20T21:02:00.322 INFO:teuthology.orchestra.run.trial199.stdout: cryptsetup x86_64 2.8.1-3.el9 baseos 351 k 2026-02-20T21:02:00.322 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:00.322 INFO:teuthology.orchestra.run.trial199.stdout:Transaction Summary 2026-02-20T21:02:00.322 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:02:00.322 INFO:teuthology.orchestra.run.trial199.stdout:Install 2 Packages 2026-02-20T21:02:00.322 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:00.322 INFO:teuthology.orchestra.run.trial199.stdout:Total download size: 648 k 2026-02-20T21:02:00.323 INFO:teuthology.orchestra.run.trial199.stdout:Installed size: 2.2 M 2026-02-20T21:02:00.323 INFO:teuthology.orchestra.run.trial199.stdout:Downloading Packages: 2026-02-20T21:02:00.684 INFO:teuthology.orchestra.run.trial199.stdout:(1/2): cryptsetup-2.8.1-3.el9.x86_64.rpm 5.9 MB/s | 351 kB 00:00 2026-02-20T21:02:00.733 INFO:teuthology.orchestra.run.trial199.stdout:(2/2): ceph-volume-20.2.0-677.gf7870454.el9.noa 2.7 MB/s | 297 kB 00:00 2026-02-20T21:02:00.734 INFO:teuthology.orchestra.run.trial199.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:02:00.734 INFO:teuthology.orchestra.run.trial199.stdout:Total 1.5 MB/s | 648 kB 00:00 2026-02-20T21:02:00.742 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction check 2026-02-20T21:02:00.748 INFO:teuthology.orchestra.run.trial199.stdout:Transaction check succeeded. 2026-02-20T21:02:00.748 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction test 2026-02-20T21:02:00.791 INFO:teuthology.orchestra.run.trial199.stdout:Transaction test succeeded. 2026-02-20T21:02:00.791 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction 2026-02-20T21:02:00.863 INFO:teuthology.orchestra.run.trial196.stdout:(1/14): ceph-prometheus-alerts-20.2.0-677.gf787 208 kB/s | 17 kB 00:00 2026-02-20T21:02:00.865 INFO:teuthology.orchestra.run.trial196.stdout:(2/14): ceph-grafana-dashboards-20.2.0-677.gf78 504 kB/s | 43 kB 00:00 2026-02-20T21:02:00.871 INFO:teuthology.orchestra.run.trial199.stdout: Preparing : 1/1 2026-02-20T21:02:00.932 INFO:teuthology.orchestra.run.trial196.stdout:(3/14): python3-jmespath-1.0.1-1.el9.noarch.rpm 714 kB/s | 48 kB 00:00 2026-02-20T21:02:00.933 INFO:teuthology.orchestra.run.trial199.stdout: Installing : cryptsetup-2.8.1-3.el9.x86_64 1/2 2026-02-20T21:02:00.935 INFO:teuthology.orchestra.run.trial199.stdout: Installing : ceph-volume-2:20.2.0-677.gf7870454.el9.noarch 2/2 2026-02-20T21:02:00.951 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: ceph-volume-2:20.2.0-677.gf7870454.el9.noarch 2/2 2026-02-20T21:02:00.951 INFO:teuthology.orchestra.run.trial199.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:02:00.951 INFO:teuthology.orchestra.run.trial199.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-02-20T21:02:00.951 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:00.995 INFO:teuthology.orchestra.run.trial196.stdout:(4/14): protobuf-3.14.0-17.el9.x86_64.rpm 7.6 MB/s | 1.0 MB 00:00 2026-02-20T21:02:00.997 INFO:teuthology.orchestra.run.trial196.stdout:(5/14): python3-protobuf-3.14.0-17.el9.noarch.r 4.0 MB/s | 267 kB 00:00 2026-02-20T21:02:01.150 INFO:teuthology.orchestra.run.trial196.stdout:(6/14): protobuf-compiler-3.14.0-17.el9.x86_64. 5.4 MB/s | 862 kB 00:00 2026-02-20T21:02:01.174 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : ceph-volume-2:20.2.0-677.gf7870454.el9.noarch 1/2 2026-02-20T21:02:01.232 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : cryptsetup-2.8.1-3.el9.x86_64 2/2 2026-02-20T21:02:01.233 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:01.233 INFO:teuthology.orchestra.run.trial199.stdout:Installed: 2026-02-20T21:02:01.233 INFO:teuthology.orchestra.run.trial199.stdout: ceph-volume-2:20.2.0-677.gf7870454.el9.noarch cryptsetup-2.8.1-3.el9.x86_64 2026-02-20T21:02:01.233 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:01.234 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:02:01.260 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install librados-devel 2026-02-20T21:02:01.350 INFO:teuthology.orchestra.run.trial196.stdout:(7/14): abseil-cpp-20211102.0-4.el9.x86_64.rpm 1.5 MB/s | 551 kB 00:00 2026-02-20T21:02:01.350 INFO:teuthology.orchestra.run.trial196.stdout:(8/14): grpc-data-1.46.7-10.el9.noarch.rpm 97 kB/s | 19 kB 00:00 2026-02-20T21:02:01.571 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:46 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:02:01.579 INFO:teuthology.orchestra.run.trial196.stdout:(9/14): ceph-mgr-dashboard-20.2.0-677.gf7870454 13 MB/s | 11 MB 00:00 2026-02-20T21:02:01.582 INFO:teuthology.orchestra.run.trial196.stdout:(10/14): python3-grpcio-1.46.7-10.el9.x86_64.rp 8.8 MB/s | 2.0 MB 00:00 2026-02-20T21:02:01.618 INFO:teuthology.orchestra.run.trial196.stdout:(11/14): python3-grpcio-tools-1.46.7-10.el9.x86 540 kB/s | 144 kB 00:00 2026-02-20T21:02:01.654 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:02:01.654 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:02:01.654 INFO:teuthology.orchestra.run.trial199.stdout: Package Arch Version Repo Size 2026-02-20T21:02:01.654 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:02:01.654 INFO:teuthology.orchestra.run.trial199.stdout:Installing: 2026-02-20T21:02:01.655 INFO:teuthology.orchestra.run.trial199.stdout: librados-devel x86_64 2:20.2.0-677.gf7870454.el9 ceph 126 k 2026-02-20T21:02:01.655 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:01.655 INFO:teuthology.orchestra.run.trial199.stdout:Transaction Summary 2026-02-20T21:02:01.655 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:02:01.655 INFO:teuthology.orchestra.run.trial199.stdout:Install 1 Package 2026-02-20T21:02:01.655 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:01.655 INFO:teuthology.orchestra.run.trial199.stdout:Total download size: 126 k 2026-02-20T21:02:01.655 INFO:teuthology.orchestra.run.trial199.stdout:Installed size: 449 k 2026-02-20T21:02:01.655 INFO:teuthology.orchestra.run.trial199.stdout:Downloading Packages: 2026-02-20T21:02:01.662 INFO:teuthology.orchestra.run.trial196.stdout:(12/14): python3-repoze-lru-0.7-16.el9.noarch.r 372 kB/s | 31 kB 00:00 2026-02-20T21:02:01.682 INFO:teuthology.orchestra.run.trial196.stdout:(13/14): python3-routes-2.5.1-5.el9.noarch.rpm 1.8 MB/s | 188 kB 00:00 2026-02-20T21:02:01.695 INFO:teuthology.orchestra.run.trial196.stdout:(14/14): python3-xmltodict-0.12.0-15.el9.noarch 286 kB/s | 22 kB 00:00 2026-02-20T21:02:01.697 INFO:teuthology.orchestra.run.trial196.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:02:01.697 INFO:teuthology.orchestra.run.trial196.stdout:Total 10 MB/s | 16 MB 00:01 2026-02-20T21:02:01.731 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction check 2026-02-20T21:02:01.740 INFO:teuthology.orchestra.run.trial196.stdout:Transaction check succeeded. 2026-02-20T21:02:01.740 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction test 2026-02-20T21:02:01.740 INFO:teuthology.orchestra.run.trial199.stdout:librados-devel-20.2.0-677.gf7870454.el9.x86_64. 1.4 MB/s | 126 kB 00:00 2026-02-20T21:02:01.741 INFO:teuthology.orchestra.run.trial199.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:02:01.742 INFO:teuthology.orchestra.run.trial199.stdout:Total 1.4 MB/s | 126 kB 00:00 2026-02-20T21:02:01.742 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction check 2026-02-20T21:02:01.744 INFO:teuthology.orchestra.run.trial199.stdout:Transaction check succeeded. 2026-02-20T21:02:01.744 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction test 2026-02-20T21:02:01.775 INFO:teuthology.orchestra.run.trial199.stdout:Transaction test succeeded. 2026-02-20T21:02:01.775 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction 2026-02-20T21:02:01.829 INFO:teuthology.orchestra.run.trial199.stdout: Preparing : 1/1 2026-02-20T21:02:01.834 INFO:teuthology.orchestra.run.trial196.stdout:Transaction test succeeded. 2026-02-20T21:02:01.834 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction 2026-02-20T21:02:01.846 INFO:teuthology.orchestra.run.trial199.stdout: Installing : librados-devel-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:01.937 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: librados-devel-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:01.951 INFO:teuthology.orchestra.run.trial196.stdout: Preparing : 1/1 2026-02-20T21:02:01.959 INFO:teuthology.orchestra.run.trial196.stdout: Installing : protobuf-3.14.0-17.el9.x86_64 1/14 2026-02-20T21:02:01.962 INFO:teuthology.orchestra.run.trial196.stdout: Installing : protobuf-compiler-3.14.0-17.el9.x86_64 2/14 2026-02-20T21:02:01.967 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-xmltodict-0.12.0-15.el9.noarch 3/14 2026-02-20T21:02:01.979 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 4/14 2026-02-20T21:02:01.981 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 5/14 2026-02-20T21:02:01.995 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : librados-devel-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:01.995 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:01.995 INFO:teuthology.orchestra.run.trial199.stdout:Installed: 2026-02-20T21:02:01.995 INFO:teuthology.orchestra.run.trial199.stdout: librados-devel-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:02:01.995 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:01.995 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:02:02.009 INFO:teuthology.orchestra.run.trial196.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 6/14 2026-02-20T21:02:02.022 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install libcephfs2 2026-02-20T21:02:02.052 INFO:teuthology.orchestra.run.trial196.stdout: Installing : abseil-cpp-20211102.0-4.el9.x86_64 7/14 2026-02-20T21:02:02.082 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 8/14 2026-02-20T21:02:02.091 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-protobuf-3.14.0-17.el9.noarch 9/14 2026-02-20T21:02:02.099 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 10/14 2026-02-20T21:02:02.101 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-jmespath-1.0.1-1.el9.noarch 11/14 2026-02-20T21:02:02.108 INFO:teuthology.orchestra.run.trial196.stdout: Installing : ceph-prometheus-alerts-2:20.2.0-677.gf7870454.el9. 12/14 2026-02-20T21:02:02.334 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:47 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:02:02.400 INFO:teuthology.orchestra.run.trial199.stdout:Package libcephfs2-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:02:02.418 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:02:02.419 INFO:teuthology.orchestra.run.trial199.stdout:Nothing to do. 2026-02-20T21:02:02.419 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:02:02.440 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install libcephfs-devel 2026-02-20T21:02:02.756 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:47 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:02:02.841 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:02:02.841 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:02:02.841 INFO:teuthology.orchestra.run.trial199.stdout: Package Arch Version Repo Size 2026-02-20T21:02:02.842 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:02:02.842 INFO:teuthology.orchestra.run.trial199.stdout:Installing: 2026-02-20T21:02:02.842 INFO:teuthology.orchestra.run.trial199.stdout: libcephfs-devel x86_64 2:20.2.0-677.gf7870454.el9 ceph 34 k 2026-02-20T21:02:02.842 INFO:teuthology.orchestra.run.trial199.stdout:Installing dependencies: 2026-02-20T21:02:02.842 INFO:teuthology.orchestra.run.trial199.stdout: libcephfs-proxy2 x86_64 2:20.2.0-677.gf7870454.el9 ceph 24 k 2026-02-20T21:02:02.842 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:02.842 INFO:teuthology.orchestra.run.trial199.stdout:Transaction Summary 2026-02-20T21:02:02.842 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:02:02.842 INFO:teuthology.orchestra.run.trial199.stdout:Install 2 Packages 2026-02-20T21:02:02.842 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:02.843 INFO:teuthology.orchestra.run.trial199.stdout:Total download size: 58 k 2026-02-20T21:02:02.843 INFO:teuthology.orchestra.run.trial199.stdout:Installed size: 207 k 2026-02-20T21:02:02.843 INFO:teuthology.orchestra.run.trial199.stdout:Downloading Packages: 2026-02-20T21:02:02.905 INFO:teuthology.orchestra.run.trial199.stdout:(1/2): libcephfs-devel-20.2.0-677.gf7870454.el9 541 kB/s | 34 kB 00:00 2026-02-20T21:02:02.907 INFO:teuthology.orchestra.run.trial199.stdout:(2/2): libcephfs-proxy2-20.2.0-677.gf7870454.el 368 kB/s | 24 kB 00:00 2026-02-20T21:02:02.908 INFO:teuthology.orchestra.run.trial199.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:02:02.909 INFO:teuthology.orchestra.run.trial199.stdout:Total 873 kB/s | 58 kB 00:00 2026-02-20T21:02:02.909 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction check 2026-02-20T21:02:02.911 INFO:teuthology.orchestra.run.trial199.stdout:Transaction check succeeded. 2026-02-20T21:02:02.912 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction test 2026-02-20T21:02:02.943 INFO:teuthology.orchestra.run.trial199.stdout:Transaction test succeeded. 2026-02-20T21:02:02.943 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction 2026-02-20T21:02:02.991 INFO:teuthology.orchestra.run.trial199.stdout: Preparing : 1/1 2026-02-20T21:02:02.992 INFO:teuthology.orchestra.run.trial199.stdout: Installing : libcephfs-proxy2-2:20.2.0-677.gf7870454.el9.x86_64 1/2 2026-02-20T21:02:03.006 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: libcephfs-proxy2-2:20.2.0-677.gf7870454.el9.x86_64 1/2 2026-02-20T21:02:03.022 INFO:teuthology.orchestra.run.trial199.stdout: Installing : libcephfs-devel-2:20.2.0-677.gf7870454.el9.x86_64 2/2 2026-02-20T21:02:03.080 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: libcephfs-devel-2:20.2.0-677.gf7870454.el9.x86_64 2/2 2026-02-20T21:02:03.080 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : libcephfs-devel-2:20.2.0-677.gf7870454.el9.x86_64 1/2 2026-02-20T21:02:03.138 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : libcephfs-proxy2-2:20.2.0-677.gf7870454.el9.x86_64 2/2 2026-02-20T21:02:03.138 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:03.139 INFO:teuthology.orchestra.run.trial199.stdout:Installed: 2026-02-20T21:02:03.139 INFO:teuthology.orchestra.run.trial199.stdout: libcephfs-devel-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:02:03.139 INFO:teuthology.orchestra.run.trial199.stdout: libcephfs-proxy2-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:02:03.139 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:03.139 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:02:03.165 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install librados2 2026-02-20T21:02:03.251 INFO:teuthology.orchestra.run.trial196.stdout: Installing : ceph-grafana-dashboards-2:20.2.0-677.gf7870454.el9 13/14 2026-02-20T21:02:03.256 INFO:teuthology.orchestra.run.trial196.stdout: Installing : ceph-mgr-dashboard-2:20.2.0-677.gf7870454.el9.noar 14/14 2026-02-20T21:02:03.458 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: ceph-mgr-dashboard-2:20.2.0-677.gf7870454.el9.noar 14/14 2026-02-20T21:02:03.459 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : ceph-grafana-dashboards-2:20.2.0-677.gf7870454.el9 1/14 2026-02-20T21:02:03.459 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : ceph-mgr-dashboard-2:20.2.0-677.gf7870454.el9.noar 2/14 2026-02-20T21:02:03.459 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : ceph-prometheus-alerts-2:20.2.0-677.gf7870454.el9. 3/14 2026-02-20T21:02:03.459 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 4/14 2026-02-20T21:02:03.459 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-jmespath-1.0.1-1.el9.noarch 5/14 2026-02-20T21:02:03.459 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 6/14 2026-02-20T21:02:03.459 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 7/14 2026-02-20T21:02:03.459 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 8/14 2026-02-20T21:02:03.459 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 9/14 2026-02-20T21:02:03.460 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 10/14 2026-02-20T21:02:03.460 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 11/14 2026-02-20T21:02:03.460 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 12/14 2026-02-20T21:02:03.460 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 13/14 2026-02-20T21:02:03.476 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:48 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:02:03.513 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-xmltodict-0.12.0-15.el9.noarch 14/14 2026-02-20T21:02:03.513 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:03.513 INFO:teuthology.orchestra.run.trial196.stdout:Installed: 2026-02-20T21:02:03.513 INFO:teuthology.orchestra.run.trial196.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-02-20T21:02:03.513 INFO:teuthology.orchestra.run.trial196.stdout: ceph-grafana-dashboards-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:02:03.514 INFO:teuthology.orchestra.run.trial196.stdout: ceph-mgr-dashboard-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:02:03.514 INFO:teuthology.orchestra.run.trial196.stdout: ceph-prometheus-alerts-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:02:03.514 INFO:teuthology.orchestra.run.trial196.stdout: grpc-data-1.46.7-10.el9.noarch 2026-02-20T21:02:03.514 INFO:teuthology.orchestra.run.trial196.stdout: protobuf-3.14.0-17.el9.x86_64 2026-02-20T21:02:03.514 INFO:teuthology.orchestra.run.trial196.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-02-20T21:02:03.514 INFO:teuthology.orchestra.run.trial196.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-02-20T21:02:03.514 INFO:teuthology.orchestra.run.trial196.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-02-20T21:02:03.514 INFO:teuthology.orchestra.run.trial196.stdout: python3-jmespath-1.0.1-1.el9.noarch 2026-02-20T21:02:03.514 INFO:teuthology.orchestra.run.trial196.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-02-20T21:02:03.514 INFO:teuthology.orchestra.run.trial196.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-02-20T21:02:03.514 INFO:teuthology.orchestra.run.trial196.stdout: python3-routes-2.5.1-5.el9.noarch 2026-02-20T21:02:03.515 INFO:teuthology.orchestra.run.trial196.stdout: python3-xmltodict-0.12.0-15.el9.noarch 2026-02-20T21:02:03.515 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:03.515 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:02:03.541 INFO:teuthology.orchestra.run.trial199.stdout:Package librados2-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:02:03.545 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install ceph-mgr-diskprediction-local 2026-02-20T21:02:03.559 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:02:03.559 INFO:teuthology.orchestra.run.trial199.stdout:Nothing to do. 2026-02-20T21:02:03.559 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:02:03.581 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install librbd1 2026-02-20T21:02:03.850 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:40 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:02:03.892 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:48 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:02:03.935 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:02:03.935 INFO:teuthology.orchestra.run.trial196.stdout:====================================================================================== 2026-02-20T21:02:03.935 INFO:teuthology.orchestra.run.trial196.stdout: Package Arch Version Repository Size 2026-02-20T21:02:03.935 INFO:teuthology.orchestra.run.trial196.stdout:====================================================================================== 2026-02-20T21:02:03.935 INFO:teuthology.orchestra.run.trial196.stdout:Installing: 2026-02-20T21:02:03.935 INFO:teuthology.orchestra.run.trial196.stdout: ceph-mgr-diskprediction-local noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 7.4 M 2026-02-20T21:02:03.935 INFO:teuthology.orchestra.run.trial196.stdout:Installing dependencies: 2026-02-20T21:02:03.935 INFO:teuthology.orchestra.run.trial196.stdout: flexiblas x86_64 3.0.4-9.el9 appstream 30 k 2026-02-20T21:02:03.935 INFO:teuthology.orchestra.run.trial196.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 appstream 3.0 M 2026-02-20T21:02:03.935 INFO:teuthology.orchestra.run.trial196.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 appstream 15 k 2026-02-20T21:02:03.935 INFO:teuthology.orchestra.run.trial196.stdout: libgfortran x86_64 11.5.0-14.el9 baseos 794 k 2026-02-20T21:02:03.935 INFO:teuthology.orchestra.run.trial196.stdout: libquadmath x86_64 11.5.0-14.el9 baseos 184 k 2026-02-20T21:02:03.936 INFO:teuthology.orchestra.run.trial196.stdout: openblas x86_64 0.3.29-1.el9 appstream 42 k 2026-02-20T21:02:03.936 INFO:teuthology.orchestra.run.trial196.stdout: openblas-openmp x86_64 0.3.29-1.el9 appstream 5.3 M 2026-02-20T21:02:03.936 INFO:teuthology.orchestra.run.trial196.stdout: python3-devel x86_64 3.9.25-3.el9 appstream 244 k 2026-02-20T21:02:03.936 INFO:teuthology.orchestra.run.trial196.stdout: python3-numpy x86_64 1:1.23.5-2.el9 appstream 6.1 M 2026-02-20T21:02:03.936 INFO:teuthology.orchestra.run.trial196.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 appstream 442 k 2026-02-20T21:02:03.936 INFO:teuthology.orchestra.run.trial196.stdout: python3-scipy x86_64 1.9.3-2.el9 appstream 19 M 2026-02-20T21:02:03.936 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:03.936 INFO:teuthology.orchestra.run.trial196.stdout:Transaction Summary 2026-02-20T21:02:03.936 INFO:teuthology.orchestra.run.trial196.stdout:====================================================================================== 2026-02-20T21:02:03.936 INFO:teuthology.orchestra.run.trial196.stdout:Install 12 Packages 2026-02-20T21:02:03.936 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:03.936 INFO:teuthology.orchestra.run.trial196.stdout:Total download size: 43 M 2026-02-20T21:02:03.936 INFO:teuthology.orchestra.run.trial196.stdout:Installed size: 234 M 2026-02-20T21:02:03.936 INFO:teuthology.orchestra.run.trial196.stdout:Downloading Packages: 2026-02-20T21:02:03.956 INFO:teuthology.orchestra.run.trial199.stdout:Package librbd1-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:02:03.974 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:02:03.975 INFO:teuthology.orchestra.run.trial199.stdout:Nothing to do. 2026-02-20T21:02:03.975 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:02:03.995 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install python3-rados 2026-02-20T21:02:04.204 INFO:teuthology.orchestra.run.trial196.stdout:(1/12): libquadmath-11.5.0-14.el9.x86_64.rpm 1.2 MB/s | 184 kB 00:00 2026-02-20T21:02:04.255 INFO:teuthology.orchestra.run.trial196.stdout:(2/12): libgfortran-11.5.0-14.el9.x86_64.rpm 3.9 MB/s | 794 kB 00:00 2026-02-20T21:02:04.270 INFO:teuthology.orchestra.run.trial196.stdout:(3/12): flexiblas-3.0.4-9.el9.x86_64.rpm 449 kB/s | 30 kB 00:00 2026-02-20T21:02:04.306 INFO:teuthology.orchestra.run.trial196.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-9.el9.x 426 kB/s | 15 kB 00:00 2026-02-20T21:02:04.308 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:49 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:02:04.319 INFO:teuthology.orchestra.run.trial196.stdout:(5/12): flexiblas-netlib-3.0.4-9.el9.x86_64.rpm 47 MB/s | 3.0 MB 00:00 2026-02-20T21:02:04.363 INFO:teuthology.orchestra.run.trial196.stdout:(6/12): openblas-0.3.29-1.el9.x86_64.rpm 736 kB/s | 42 kB 00:00 2026-02-20T21:02:04.373 INFO:teuthology.orchestra.run.trial199.stdout:Package python3-rados-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:02:04.383 INFO:teuthology.orchestra.run.trial196.stdout:(7/12): openblas-openmp-0.3.29-1.el9.x86_64.rpm 83 MB/s | 5.3 MB 00:00 2026-02-20T21:02:04.391 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:02:04.391 INFO:teuthology.orchestra.run.trial199.stdout:Nothing to do. 2026-02-20T21:02:04.391 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:02:04.401 INFO:teuthology.orchestra.run.trial196.stdout:(8/12): python3-devel-3.9.25-3.el9.x86_64.rpm 6.3 MB/s | 244 kB 00:00 2026-02-20T21:02:04.413 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install python3-rgw 2026-02-20T21:02:04.435 INFO:teuthology.orchestra.run.trial196.stdout:(9/12): python3-numpy-f2py-1.23.5-2.el9.x86_64. 13 MB/s | 442 kB 00:00 2026-02-20T21:02:04.449 INFO:teuthology.orchestra.run.trial196.stdout:(10/12): python3-numpy-1.23.5-2.el9.x86_64.rpm 93 MB/s | 6.1 MB 00:00 2026-02-20T21:02:04.595 INFO:teuthology.orchestra.run.trial196.stdout:(11/12): python3-scipy-1.9.3-2.el9.x86_64.rpm 120 MB/s | 19 MB 00:00 2026-02-20T21:02:04.720 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:49 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:02:04.787 INFO:teuthology.orchestra.run.trial199.stdout:Package python3-rgw-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:02:04.805 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:02:04.806 INFO:teuthology.orchestra.run.trial199.stdout:Nothing to do. 2026-02-20T21:02:04.806 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:02:04.827 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install python3-cephfs 2026-02-20T21:02:05.051 INFO:teuthology.orchestra.run.trial196.stdout:(12/12): ceph-mgr-diskprediction-local-20.2.0-6 7.5 MB/s | 7.4 MB 00:00 2026-02-20T21:02:05.052 INFO:teuthology.orchestra.run.trial196.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:02:05.053 INFO:teuthology.orchestra.run.trial196.stdout:Total 38 MB/s | 43 MB 00:01 2026-02-20T21:02:05.130 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:50 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:02:05.195 INFO:teuthology.orchestra.run.trial199.stdout:Package python3-cephfs-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:02:05.210 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction check 2026-02-20T21:02:05.213 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:02:05.213 INFO:teuthology.orchestra.run.trial199.stdout:Nothing to do. 2026-02-20T21:02:05.214 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:02:05.224 INFO:teuthology.orchestra.run.trial196.stdout:Transaction check succeeded. 2026-02-20T21:02:05.224 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction test 2026-02-20T21:02:05.235 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install python3-rbd 2026-02-20T21:02:05.424 INFO:teuthology.orchestra.run.trial196.stdout:Transaction test succeeded. 2026-02-20T21:02:05.424 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction 2026-02-20T21:02:05.541 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:50 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:02:05.607 INFO:teuthology.orchestra.run.trial199.stdout:Package python3-rbd-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:02:05.625 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:02:05.626 INFO:teuthology.orchestra.run.trial199.stdout:Nothing to do. 2026-02-20T21:02:05.626 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:02:05.638 INFO:teuthology.orchestra.run.trial196.stdout: Preparing : 1/1 2026-02-20T21:02:05.641 INFO:teuthology.orchestra.run.trial196.stdout: Installing : flexiblas-3.0.4-9.el9.x86_64 1/12 2026-02-20T21:02:05.649 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install rbd-fuse 2026-02-20T21:02:05.650 INFO:teuthology.orchestra.run.trial196.stdout: Installing : libquadmath-11.5.0-14.el9.x86_64 2/12 2026-02-20T21:02:05.681 INFO:teuthology.orchestra.run.trial196.stdout: Installing : libgfortran-11.5.0-14.el9.x86_64 3/12 2026-02-20T21:02:05.685 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-devel-3.9.25-3.el9.x86_64 4/12 2026-02-20T21:02:05.744 INFO:teuthology.orchestra.run.trial196.stdout: Installing : openblas-0.3.29-1.el9.x86_64 5/12 2026-02-20T21:02:05.747 INFO:teuthology.orchestra.run.trial196.stdout: Installing : openblas-openmp-0.3.29-1.el9.x86_64 6/12 2026-02-20T21:02:05.768 INFO:teuthology.orchestra.run.trial196.stdout: Installing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 7/12 2026-02-20T21:02:05.958 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:50 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:02:06.041 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:02:06.041 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:02:06.041 INFO:teuthology.orchestra.run.trial199.stdout: Package Architecture Version Repository Size 2026-02-20T21:02:06.042 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:02:06.042 INFO:teuthology.orchestra.run.trial199.stdout:Installing: 2026-02-20T21:02:06.042 INFO:teuthology.orchestra.run.trial199.stdout: rbd-fuse x86_64 2:20.2.0-677.gf7870454.el9 ceph 91 k 2026-02-20T21:02:06.042 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:06.042 INFO:teuthology.orchestra.run.trial199.stdout:Transaction Summary 2026-02-20T21:02:06.042 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:02:06.042 INFO:teuthology.orchestra.run.trial199.stdout:Install 1 Package 2026-02-20T21:02:06.042 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:06.042 INFO:teuthology.orchestra.run.trial199.stdout:Total download size: 91 k 2026-02-20T21:02:06.042 INFO:teuthology.orchestra.run.trial199.stdout:Installed size: 238 k 2026-02-20T21:02:06.042 INFO:teuthology.orchestra.run.trial199.stdout:Downloading Packages: 2026-02-20T21:02:06.093 INFO:teuthology.orchestra.run.trial196.stdout: Installing : flexiblas-netlib-3.0.4-9.el9.x86_64 8/12 2026-02-20T21:02:06.125 INFO:teuthology.orchestra.run.trial199.stdout:rbd-fuse-20.2.0-677.gf7870454.el9.x86_64.rpm 1.1 MB/s | 91 kB 00:00 2026-02-20T21:02:06.126 INFO:teuthology.orchestra.run.trial199.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:02:06.126 INFO:teuthology.orchestra.run.trial199.stdout:Total 1.1 MB/s | 91 kB 00:00 2026-02-20T21:02:06.126 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction check 2026-02-20T21:02:06.130 INFO:teuthology.orchestra.run.trial199.stdout:Transaction check succeeded. 2026-02-20T21:02:06.130 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction test 2026-02-20T21:02:06.161 INFO:teuthology.orchestra.run.trial199.stdout:Transaction test succeeded. 2026-02-20T21:02:06.162 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction 2026-02-20T21:02:06.165 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-numpy-1:1.23.5-2.el9.x86_64 9/12 2026-02-20T21:02:06.212 INFO:teuthology.orchestra.run.trial199.stdout: Preparing : 1/1 2026-02-20T21:02:06.228 INFO:teuthology.orchestra.run.trial199.stdout: Installing : rbd-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:06.310 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: rbd-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:06.369 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : rbd-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:06.369 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:06.369 INFO:teuthology.orchestra.run.trial199.stdout:Installed: 2026-02-20T21:02:06.369 INFO:teuthology.orchestra.run.trial199.stdout: rbd-fuse-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:02:06.369 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:06.369 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:02:06.399 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install rbd-mirror 2026-02-20T21:02:06.710 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:51 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:02:06.795 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:02:06.795 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:02:06.795 INFO:teuthology.orchestra.run.trial199.stdout: Package Arch Version Repo Size 2026-02-20T21:02:06.795 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:02:06.795 INFO:teuthology.orchestra.run.trial199.stdout:Installing: 2026-02-20T21:02:06.795 INFO:teuthology.orchestra.run.trial199.stdout: rbd-mirror x86_64 2:20.2.0-677.gf7870454.el9 ceph 2.9 M 2026-02-20T21:02:06.796 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:06.796 INFO:teuthology.orchestra.run.trial199.stdout:Transaction Summary 2026-02-20T21:02:06.796 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:02:06.796 INFO:teuthology.orchestra.run.trial199.stdout:Install 1 Package 2026-02-20T21:02:06.796 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:06.796 INFO:teuthology.orchestra.run.trial199.stdout:Total download size: 2.9 M 2026-02-20T21:02:06.796 INFO:teuthology.orchestra.run.trial199.stdout:Installed size: 11 M 2026-02-20T21:02:06.796 INFO:teuthology.orchestra.run.trial199.stdout:Downloading Packages: 2026-02-20T21:02:06.835 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 10/12 2026-02-20T21:02:07.284 INFO:teuthology.orchestra.run.trial199.stdout:rbd-mirror-20.2.0-677.gf7870454.el9.x86_64.rpm 6.0 MB/s | 2.9 MB 00:00 2026-02-20T21:02:07.284 INFO:teuthology.orchestra.run.trial199.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:02:07.284 INFO:teuthology.orchestra.run.trial199.stdout:Total 6.0 MB/s | 2.9 MB 00:00 2026-02-20T21:02:07.284 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction check 2026-02-20T21:02:07.289 INFO:teuthology.orchestra.run.trial199.stdout:Transaction check succeeded. 2026-02-20T21:02:07.289 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction test 2026-02-20T21:02:07.303 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-scipy-1.9.3-2.el9.x86_64 11/12 2026-02-20T21:02:07.303 INFO:teuthology.orchestra.run.trial196.stdout: Installing : ceph-mgr-diskprediction-local-2:20.2.0-677.gf78704 12/12 2026-02-20T21:02:07.327 INFO:teuthology.orchestra.run.trial199.stdout:Transaction test succeeded. 2026-02-20T21:02:07.328 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction 2026-02-20T21:02:07.514 INFO:teuthology.orchestra.run.trial199.stdout: Preparing : 1/1 2026-02-20T21:02:07.516 INFO:teuthology.orchestra.run.trial199.stdout: Installing : rbd-mirror-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:07.539 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: rbd-mirror-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:07.540 INFO:teuthology.orchestra.run.trial199.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:02:07.540 INFO:teuthology.orchestra.run.trial199.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-02-20T21:02:07.540 INFO:teuthology.orchestra.run.trial199.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-20T21:02:07.540 INFO:teuthology.orchestra.run.trial199.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-20T21:02:07.540 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:07.570 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:20.2.0-677.gf78704 12/12 2026-02-20T21:02:07.571 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : ceph-mgr-diskprediction-local-2:20.2.0-677.gf78704 1/12 2026-02-20T21:02:07.571 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 2/12 2026-02-20T21:02:07.571 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 3/12 2026-02-20T21:02:07.571 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 4/12 2026-02-20T21:02:07.571 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 5/12 2026-02-20T21:02:07.571 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 6/12 2026-02-20T21:02:07.571 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 7/12 2026-02-20T21:02:07.571 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 8/12 2026-02-20T21:02:07.571 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-devel-3.9.25-3.el9.x86_64 9/12 2026-02-20T21:02:07.571 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 10/12 2026-02-20T21:02:07.572 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 11/12 2026-02-20T21:02:07.628 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 12/12 2026-02-20T21:02:07.628 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:07.628 INFO:teuthology.orchestra.run.trial196.stdout:Installed: 2026-02-20T21:02:07.628 INFO:teuthology.orchestra.run.trial196.stdout: ceph-mgr-diskprediction-local-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:02:07.628 INFO:teuthology.orchestra.run.trial196.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-02-20T21:02:07.628 INFO:teuthology.orchestra.run.trial196.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-02-20T21:02:07.628 INFO:teuthology.orchestra.run.trial196.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-02-20T21:02:07.628 INFO:teuthology.orchestra.run.trial196.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-02-20T21:02:07.628 INFO:teuthology.orchestra.run.trial196.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-02-20T21:02:07.628 INFO:teuthology.orchestra.run.trial196.stdout: openblas-0.3.29-1.el9.x86_64 2026-02-20T21:02:07.628 INFO:teuthology.orchestra.run.trial196.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-02-20T21:02:07.628 INFO:teuthology.orchestra.run.trial196.stdout: python3-devel-3.9.25-3.el9.x86_64 2026-02-20T21:02:07.628 INFO:teuthology.orchestra.run.trial196.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-02-20T21:02:07.628 INFO:teuthology.orchestra.run.trial196.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-02-20T21:02:07.628 INFO:teuthology.orchestra.run.trial196.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-02-20T21:02:07.628 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:07.628 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:02:07.667 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install ceph-mgr-rook 2026-02-20T21:02:07.833 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : rbd-mirror-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:07.834 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:07.834 INFO:teuthology.orchestra.run.trial199.stdout:Installed: 2026-02-20T21:02:07.834 INFO:teuthology.orchestra.run.trial199.stdout: rbd-mirror-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:02:07.834 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:07.834 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:02:07.862 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install rbd-nbd 2026-02-20T21:02:07.979 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:44 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:02:08.064 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:02:08.064 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:08.064 INFO:teuthology.orchestra.run.trial196.stdout: Package Arch Version Repository Size 2026-02-20T21:02:08.064 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:08.064 INFO:teuthology.orchestra.run.trial196.stdout:Installing: 2026-02-20T21:02:08.064 INFO:teuthology.orchestra.run.trial196.stdout: ceph-mgr-rook noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 50 k 2026-02-20T21:02:08.064 INFO:teuthology.orchestra.run.trial196.stdout:Installing dependencies: 2026-02-20T21:02:08.064 INFO:teuthology.orchestra.run.trial196.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2026-02-20T21:02:08.064 INFO:teuthology.orchestra.run.trial196.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2026-02-20T21:02:08.064 INFO:teuthology.orchestra.run.trial196.stdout: python3-google-auth noarch 1:2.45.0-1.el9 epel 254 k 2026-02-20T21:02:08.064 INFO:teuthology.orchestra.run.trial196.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 epel 1.0 M 2026-02-20T21:02:08.064 INFO:teuthology.orchestra.run.trial196.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2026-02-20T21:02:08.064 INFO:teuthology.orchestra.run.trial196.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2026-02-20T21:02:08.064 INFO:teuthology.orchestra.run.trial196.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2026-02-20T21:02:08.064 INFO:teuthology.orchestra.run.trial196.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2026-02-20T21:02:08.064 INFO:teuthology.orchestra.run.trial196.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2026-02-20T21:02:08.065 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:08.065 INFO:teuthology.orchestra.run.trial196.stdout:Transaction Summary 2026-02-20T21:02:08.065 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:08.065 INFO:teuthology.orchestra.run.trial196.stdout:Install 10 Packages 2026-02-20T21:02:08.065 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:08.065 INFO:teuthology.orchestra.run.trial196.stdout:Total download size: 2.0 M 2026-02-20T21:02:08.065 INFO:teuthology.orchestra.run.trial196.stdout:Installed size: 25 M 2026-02-20T21:02:08.065 INFO:teuthology.orchestra.run.trial196.stdout:Downloading Packages: 2026-02-20T21:02:08.168 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:53 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:02:08.253 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:02:08.254 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:02:08.254 INFO:teuthology.orchestra.run.trial199.stdout: Package Architecture Version Repository Size 2026-02-20T21:02:08.254 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:02:08.254 INFO:teuthology.orchestra.run.trial199.stdout:Installing: 2026-02-20T21:02:08.254 INFO:teuthology.orchestra.run.trial199.stdout: rbd-nbd x86_64 2:20.2.0-677.gf7870454.el9 ceph 180 k 2026-02-20T21:02:08.254 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:08.254 INFO:teuthology.orchestra.run.trial199.stdout:Transaction Summary 2026-02-20T21:02:08.254 INFO:teuthology.orchestra.run.trial199.stdout:================================================================================ 2026-02-20T21:02:08.254 INFO:teuthology.orchestra.run.trial199.stdout:Install 1 Package 2026-02-20T21:02:08.254 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:08.254 INFO:teuthology.orchestra.run.trial199.stdout:Total download size: 180 k 2026-02-20T21:02:08.254 INFO:teuthology.orchestra.run.trial199.stdout:Installed size: 498 k 2026-02-20T21:02:08.254 INFO:teuthology.orchestra.run.trial199.stdout:Downloading Packages: 2026-02-20T21:02:08.341 INFO:teuthology.orchestra.run.trial199.stdout:rbd-nbd-20.2.0-677.gf7870454.el9.x86_64.rpm 2.0 MB/s | 180 kB 00:00 2026-02-20T21:02:08.342 INFO:teuthology.orchestra.run.trial199.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:02:08.343 INFO:teuthology.orchestra.run.trial199.stdout:Total 2.0 MB/s | 180 kB 00:00 2026-02-20T21:02:08.343 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction check 2026-02-20T21:02:08.347 INFO:teuthology.orchestra.run.trial199.stdout:Transaction check succeeded. 2026-02-20T21:02:08.347 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction test 2026-02-20T21:02:08.379 INFO:teuthology.orchestra.run.trial199.stdout:Transaction test succeeded. 2026-02-20T21:02:08.379 INFO:teuthology.orchestra.run.trial199.stdout:Running transaction 2026-02-20T21:02:08.434 INFO:teuthology.orchestra.run.trial199.stdout: Preparing : 1/1 2026-02-20T21:02:08.450 INFO:teuthology.orchestra.run.trial199.stdout: Installing : rbd-nbd-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:08.531 INFO:teuthology.orchestra.run.trial199.stdout: Running scriptlet: rbd-nbd-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:08.589 INFO:teuthology.orchestra.run.trial199.stdout: Verifying : rbd-nbd-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:08.589 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:08.589 INFO:teuthology.orchestra.run.trial199.stdout:Installed: 2026-02-20T21:02:08.589 INFO:teuthology.orchestra.run.trial199.stdout: rbd-nbd-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:02:08.589 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:08.590 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:02:08.618 INFO:teuthology.orchestra.run.trial196.stdout:(1/10): ceph-mgr-rook-20.2.0-677.gf7870454.el9. 396 kB/s | 50 kB 00:00 2026-02-20T21:02:08.619 DEBUG:teuthology.orchestra.run.trial199:> sudo yum -y install rbd-mirror 2026-02-20T21:02:08.634 INFO:teuthology.orchestra.run.trial196.stdout:(2/10): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.1 MB/s | 159 kB 00:00 2026-02-20T21:02:08.652 INFO:teuthology.orchestra.run.trial196.stdout:(3/10): python3-pyasn1-modules-0.4.8-6.el9.noar 1.7 MB/s | 279 kB 00:00 2026-02-20T21:02:08.721 INFO:teuthology.orchestra.run.trial196.stdout:(4/10): python3-requests-oauthlib-1.3.0-12.el9. 522 kB/s | 54 kB 00:00 2026-02-20T21:02:08.852 INFO:teuthology.orchestra.run.trial196.stdout:(5/10): python3-certifi-2023.05.07-4.el9.noarch 71 kB/s | 14 kB 00:00 2026-02-20T21:02:08.907 INFO:teuthology.orchestra.run.trial196.stdout:(6/10): python3-cachetools-4.2.4-1.el9.noarch.r 118 kB/s | 32 kB 00:00 2026-02-20T21:02:08.931 INFO:teuthology.orchestra.run.trial199.stdout:Last metadata expiration check: 0:00:53 ago on Fri 20 Feb 2026 09:01:15 PM UTC. 2026-02-20T21:02:08.997 INFO:teuthology.orchestra.run.trial199.stdout:Package rbd-mirror-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:02:09.015 INFO:teuthology.orchestra.run.trial199.stdout:Dependencies resolved. 2026-02-20T21:02:09.016 INFO:teuthology.orchestra.run.trial199.stdout:Nothing to do. 2026-02-20T21:02:09.016 INFO:teuthology.orchestra.run.trial199.stdout:Complete! 2026-02-20T21:02:09.034 INFO:teuthology.orchestra.run.trial196.stdout:(7/10): python3-rsa-4.9-2.el9.noarch.rpm 462 kB/s | 59 kB 00:00 2026-02-20T21:02:09.040 DEBUG:teuthology.parallel:result is None 2026-02-20T21:02:09.130 INFO:teuthology.orchestra.run.trial196.stdout:(8/10): python3-websocket-client-1.2.3-2.el9.no 943 kB/s | 90 kB 00:00 2026-02-20T21:02:09.160 INFO:teuthology.orchestra.run.trial196.stdout:(9/10): python3-google-auth-2.45.0-1.el9.noarch 580 kB/s | 254 kB 00:00 2026-02-20T21:02:09.236 INFO:teuthology.orchestra.run.trial196.stdout:(10/10): python3-kubernetes-26.1.0-3.el9.noarch 2.7 MB/s | 1.0 MB 00:00 2026-02-20T21:02:09.237 INFO:teuthology.orchestra.run.trial196.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:02:09.238 INFO:teuthology.orchestra.run.trial196.stdout:Total 1.7 MB/s | 2.0 MB 00:01 2026-02-20T21:02:09.262 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction check 2026-02-20T21:02:09.270 INFO:teuthology.orchestra.run.trial196.stdout:Transaction check succeeded. 2026-02-20T21:02:09.270 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction test 2026-02-20T21:02:09.309 INFO:teuthology.orchestra.run.trial196.stdout:Transaction test succeeded. 2026-02-20T21:02:09.309 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction 2026-02-20T21:02:09.383 INFO:teuthology.orchestra.run.trial196.stdout: Preparing : 1/1 2026-02-20T21:02:09.433 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/10 2026-02-20T21:02:09.446 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/10 2026-02-20T21:02:09.460 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/10 2026-02-20T21:02:09.465 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/10 2026-02-20T21:02:09.473 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/10 2026-02-20T21:02:09.512 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/10 2026-02-20T21:02:09.524 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-google-auth-1:2.45.0-1.el9.noarch 7/10 2026-02-20T21:02:09.842 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 8/10 2026-02-20T21:02:09.856 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-kubernetes-1:26.1.0-3.el9.noarch 9/10 2026-02-20T21:02:09.857 INFO:teuthology.orchestra.run.trial196.stdout: Installing : ceph-mgr-rook-2:20.2.0-677.gf7870454.el9.noarch 10/10 2026-02-20T21:02:09.963 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: ceph-mgr-rook-2:20.2.0-677.gf7870454.el9.noarch 10/10 2026-02-20T21:02:09.963 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : ceph-mgr-rook-2:20.2.0-677.gf7870454.el9.noarch 1/10 2026-02-20T21:02:09.963 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 2/10 2026-02-20T21:02:09.963 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 3/10 2026-02-20T21:02:09.963 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 4/10 2026-02-20T21:02:09.964 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 5/10 2026-02-20T21:02:09.964 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 6/10 2026-02-20T21:02:09.964 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 7/10 2026-02-20T21:02:09.964 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 8/10 2026-02-20T21:02:09.964 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 9/10 2026-02-20T21:02:10.019 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 10/10 2026-02-20T21:02:10.019 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:10.019 INFO:teuthology.orchestra.run.trial196.stdout:Installed: 2026-02-20T21:02:10.019 INFO:teuthology.orchestra.run.trial196.stdout: ceph-mgr-rook-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:02:10.019 INFO:teuthology.orchestra.run.trial196.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-02-20T21:02:10.019 INFO:teuthology.orchestra.run.trial196.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-02-20T21:02:10.019 INFO:teuthology.orchestra.run.trial196.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-02-20T21:02:10.019 INFO:teuthology.orchestra.run.trial196.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-02-20T21:02:10.020 INFO:teuthology.orchestra.run.trial196.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2026-02-20T21:02:10.020 INFO:teuthology.orchestra.run.trial196.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2026-02-20T21:02:10.020 INFO:teuthology.orchestra.run.trial196.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-02-20T21:02:10.020 INFO:teuthology.orchestra.run.trial196.stdout: python3-rsa-4.9-2.el9.noarch 2026-02-20T21:02:10.020 INFO:teuthology.orchestra.run.trial196.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-02-20T21:02:10.020 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:10.020 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:02:10.050 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install ceph-mgr-cephadm 2026-02-20T21:02:10.355 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:47 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:02:10.439 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:02:10.440 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:10.440 INFO:teuthology.orchestra.run.trial196.stdout: Package Arch Version Repository Size 2026-02-20T21:02:10.440 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:10.440 INFO:teuthology.orchestra.run.trial196.stdout:Installing: 2026-02-20T21:02:10.440 INFO:teuthology.orchestra.run.trial196.stdout: ceph-mgr-cephadm noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 173 k 2026-02-20T21:02:10.440 INFO:teuthology.orchestra.run.trial196.stdout:Installing dependencies: 2026-02-20T21:02:10.441 INFO:teuthology.orchestra.run.trial196.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2026-02-20T21:02:10.441 INFO:teuthology.orchestra.run.trial196.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2026-02-20T21:02:10.441 INFO:teuthology.orchestra.run.trial196.stdout: python3-jinja2 noarch 2.11.3-8.el9 appstream 249 k 2026-02-20T21:02:10.441 INFO:teuthology.orchestra.run.trial196.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2026-02-20T21:02:10.441 INFO:teuthology.orchestra.run.trial196.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2026-02-20T21:02:10.441 INFO:teuthology.orchestra.run.trial196.stdout: python3-typing-extensions noarch 4.15.0-1.el9 epel 86 k 2026-02-20T21:02:10.441 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:10.441 INFO:teuthology.orchestra.run.trial196.stdout:Transaction Summary 2026-02-20T21:02:10.441 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:10.441 INFO:teuthology.orchestra.run.trial196.stdout:Install 7 Packages 2026-02-20T21:02:10.441 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:10.441 INFO:teuthology.orchestra.run.trial196.stdout:Total download size: 7.1 M 2026-02-20T21:02:10.442 INFO:teuthology.orchestra.run.trial196.stdout:Installed size: 33 M 2026-02-20T21:02:10.442 INFO:teuthology.orchestra.run.trial196.stdout:Downloading Packages: 2026-02-20T21:02:10.689 INFO:teuthology.orchestra.run.trial196.stdout:(1/7): ceph-mgr-cephadm-20.2.0-677.gf7870454.el 2.0 MB/s | 173 kB 00:00 2026-02-20T21:02:11.254 INFO:teuthology.orchestra.run.trial196.stdout:(2/7): python3-markupsafe-1.1.1-12.el9.x86_64.r 61 kB/s | 35 kB 00:00 2026-02-20T21:02:11.382 INFO:teuthology.orchestra.run.trial196.stdout:(3/7): python3-jinja2-2.11.3-8.el9.noarch.rpm 320 kB/s | 249 kB 00:00 2026-02-20T21:02:11.676 INFO:teuthology.orchestra.run.trial196.stdout:(4/7): python3-natsort-7.1.1-5.el9.noarch.rpm 196 kB/s | 58 kB 00:00 2026-02-20T21:02:11.689 INFO:teuthology.orchestra.run.trial196.stdout:(5/7): python3-babel-2.9.1-2.el9.noarch.rpm 5.5 MB/s | 6.0 MB 00:01 2026-02-20T21:02:11.792 INFO:teuthology.orchestra.run.trial196.stdout:(6/7): python3-asyncssh-2.13.2-5.el9.noarch.rpm 1.0 MB/s | 548 kB 00:00 2026-02-20T21:02:11.824 INFO:teuthology.orchestra.run.trial196.stdout:(7/7): python3-typing-extensions-4.15.0-1.el9.n 584 kB/s | 86 kB 00:00 2026-02-20T21:02:11.825 INFO:teuthology.orchestra.run.trial196.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:02:11.825 INFO:teuthology.orchestra.run.trial196.stdout:Total 5.1 MB/s | 7.1 MB 00:01 2026-02-20T21:02:11.865 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction check 2026-02-20T21:02:11.871 INFO:teuthology.orchestra.run.trial196.stdout:Transaction check succeeded. 2026-02-20T21:02:11.871 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction test 2026-02-20T21:02:11.942 INFO:teuthology.orchestra.run.trial196.stdout:Transaction test succeeded. 2026-02-20T21:02:11.942 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction 2026-02-20T21:02:12.032 INFO:teuthology.orchestra.run.trial196.stdout: Preparing : 1/1 2026-02-20T21:02:12.081 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-typing-extensions-4.15.0-1.el9.noarch 1/7 2026-02-20T21:02:12.090 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/7 2026-02-20T21:02:12.096 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/7 2026-02-20T21:02:12.315 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 4/7 2026-02-20T21:02:12.339 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 5/7 2026-02-20T21:02:12.362 INFO:teuthology.orchestra.run.trial196.stdout: Installing : python3-jinja2-2.11.3-8.el9.noarch 6/7 2026-02-20T21:02:12.363 INFO:teuthology.orchestra.run.trial196.stdout: Installing : ceph-mgr-cephadm-2:20.2.0-677.gf7870454.el9.noarch 7/7 2026-02-20T21:02:12.470 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: ceph-mgr-cephadm-2:20.2.0-677.gf7870454.el9.noarch 7/7 2026-02-20T21:02:12.470 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : ceph-mgr-cephadm-2:20.2.0-677.gf7870454.el9.noarch 1/7 2026-02-20T21:02:12.470 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/7 2026-02-20T21:02:12.470 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 3/7 2026-02-20T21:02:12.470 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 4/7 2026-02-20T21:02:12.470 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 5/7 2026-02-20T21:02:12.470 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 6/7 2026-02-20T21:02:12.526 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 7/7 2026-02-20T21:02:12.526 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:12.526 INFO:teuthology.orchestra.run.trial196.stdout:Installed: 2026-02-20T21:02:12.526 INFO:teuthology.orchestra.run.trial196.stdout: ceph-mgr-cephadm-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:02:12.527 INFO:teuthology.orchestra.run.trial196.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-02-20T21:02:12.527 INFO:teuthology.orchestra.run.trial196.stdout: python3-babel-2.9.1-2.el9.noarch 2026-02-20T21:02:12.527 INFO:teuthology.orchestra.run.trial196.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-02-20T21:02:12.527 INFO:teuthology.orchestra.run.trial196.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-02-20T21:02:12.527 INFO:teuthology.orchestra.run.trial196.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-02-20T21:02:12.527 INFO:teuthology.orchestra.run.trial196.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-02-20T21:02:12.527 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:12.527 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:02:12.557 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install ceph-fuse 2026-02-20T21:02:12.862 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:49 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:02:12.945 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:02:12.946 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:12.946 INFO:teuthology.orchestra.run.trial196.stdout: Package Architecture Version Repository Size 2026-02-20T21:02:12.946 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:12.946 INFO:teuthology.orchestra.run.trial196.stdout:Installing: 2026-02-20T21:02:12.946 INFO:teuthology.orchestra.run.trial196.stdout: ceph-fuse x86_64 2:20.2.0-677.gf7870454.el9 ceph 943 k 2026-02-20T21:02:12.946 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:12.946 INFO:teuthology.orchestra.run.trial196.stdout:Transaction Summary 2026-02-20T21:02:12.946 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:12.946 INFO:teuthology.orchestra.run.trial196.stdout:Install 1 Package 2026-02-20T21:02:12.946 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:12.946 INFO:teuthology.orchestra.run.trial196.stdout:Total download size: 943 k 2026-02-20T21:02:12.946 INFO:teuthology.orchestra.run.trial196.stdout:Installed size: 2.7 M 2026-02-20T21:02:12.946 INFO:teuthology.orchestra.run.trial196.stdout:Downloading Packages: 2026-02-20T21:02:13.242 INFO:teuthology.orchestra.run.trial196.stdout:ceph-fuse-20.2.0-677.gf7870454.el9.x86_64.rpm 3.1 MB/s | 943 kB 00:00 2026-02-20T21:02:13.242 INFO:teuthology.orchestra.run.trial196.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:02:13.242 INFO:teuthology.orchestra.run.trial196.stdout:Total 3.1 MB/s | 943 kB 00:00 2026-02-20T21:02:13.243 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction check 2026-02-20T21:02:13.247 INFO:teuthology.orchestra.run.trial196.stdout:Transaction check succeeded. 2026-02-20T21:02:13.248 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction test 2026-02-20T21:02:13.281 INFO:teuthology.orchestra.run.trial196.stdout:Transaction test succeeded. 2026-02-20T21:02:13.281 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction 2026-02-20T21:02:13.370 INFO:teuthology.orchestra.run.trial196.stdout: Preparing : 1/1 2026-02-20T21:02:13.387 INFO:teuthology.orchestra.run.trial196.stdout: Installing : ceph-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:13.637 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: ceph-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:13.696 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : ceph-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:13.697 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:13.697 INFO:teuthology.orchestra.run.trial196.stdout:Installed: 2026-02-20T21:02:13.697 INFO:teuthology.orchestra.run.trial196.stdout: ceph-fuse-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:02:13.697 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:13.697 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:02:13.725 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install ceph-volume 2026-02-20T21:02:14.034 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:51 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:02:14.119 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:02:14.120 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:14.120 INFO:teuthology.orchestra.run.trial196.stdout: Package Arch Version Repository Size 2026-02-20T21:02:14.120 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:14.120 INFO:teuthology.orchestra.run.trial196.stdout:Installing: 2026-02-20T21:02:14.120 INFO:teuthology.orchestra.run.trial196.stdout: ceph-volume noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 297 k 2026-02-20T21:02:14.120 INFO:teuthology.orchestra.run.trial196.stdout:Installing dependencies: 2026-02-20T21:02:14.120 INFO:teuthology.orchestra.run.trial196.stdout: cryptsetup x86_64 2.8.1-3.el9 baseos 351 k 2026-02-20T21:02:14.121 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:14.121 INFO:teuthology.orchestra.run.trial196.stdout:Transaction Summary 2026-02-20T21:02:14.121 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:14.121 INFO:teuthology.orchestra.run.trial196.stdout:Install 2 Packages 2026-02-20T21:02:14.121 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:14.121 INFO:teuthology.orchestra.run.trial196.stdout:Total download size: 648 k 2026-02-20T21:02:14.121 INFO:teuthology.orchestra.run.trial196.stdout:Installed size: 2.2 M 2026-02-20T21:02:14.121 INFO:teuthology.orchestra.run.trial196.stdout:Downloading Packages: 2026-02-20T21:02:14.294 INFO:teuthology.orchestra.run.trial196.stdout:(1/2): ceph-volume-20.2.0-677.gf7870454.el9.noa 2.6 MB/s | 297 kB 00:00 2026-02-20T21:02:14.367 INFO:teuthology.orchestra.run.trial196.stdout:(2/2): cryptsetup-2.8.1-3.el9.x86_64.rpm 1.9 MB/s | 351 kB 00:00 2026-02-20T21:02:14.367 INFO:teuthology.orchestra.run.trial196.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:02:14.367 INFO:teuthology.orchestra.run.trial196.stdout:Total 2.6 MB/s | 648 kB 00:00 2026-02-20T21:02:14.376 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction check 2026-02-20T21:02:14.382 INFO:teuthology.orchestra.run.trial196.stdout:Transaction check succeeded. 2026-02-20T21:02:14.382 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction test 2026-02-20T21:02:14.424 INFO:teuthology.orchestra.run.trial196.stdout:Transaction test succeeded. 2026-02-20T21:02:14.424 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction 2026-02-20T21:02:14.504 INFO:teuthology.orchestra.run.trial196.stdout: Preparing : 1/1 2026-02-20T21:02:14.567 INFO:teuthology.orchestra.run.trial196.stdout: Installing : cryptsetup-2.8.1-3.el9.x86_64 1/2 2026-02-20T21:02:14.569 INFO:teuthology.orchestra.run.trial196.stdout: Installing : ceph-volume-2:20.2.0-677.gf7870454.el9.noarch 2/2 2026-02-20T21:02:14.583 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: ceph-volume-2:20.2.0-677.gf7870454.el9.noarch 2/2 2026-02-20T21:02:14.584 INFO:teuthology.orchestra.run.trial196.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:02:14.584 INFO:teuthology.orchestra.run.trial196.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-02-20T21:02:14.584 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:14.808 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : ceph-volume-2:20.2.0-677.gf7870454.el9.noarch 1/2 2026-02-20T21:02:14.867 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : cryptsetup-2.8.1-3.el9.x86_64 2/2 2026-02-20T21:02:14.867 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:14.867 INFO:teuthology.orchestra.run.trial196.stdout:Installed: 2026-02-20T21:02:14.867 INFO:teuthology.orchestra.run.trial196.stdout: ceph-volume-2:20.2.0-677.gf7870454.el9.noarch cryptsetup-2.8.1-3.el9.x86_64 2026-02-20T21:02:14.867 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:14.867 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:02:14.894 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install librados-devel 2026-02-20T21:02:15.203 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:52 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:02:15.287 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:02:15.288 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:15.288 INFO:teuthology.orchestra.run.trial196.stdout: Package Arch Version Repo Size 2026-02-20T21:02:15.288 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:15.288 INFO:teuthology.orchestra.run.trial196.stdout:Installing: 2026-02-20T21:02:15.288 INFO:teuthology.orchestra.run.trial196.stdout: librados-devel x86_64 2:20.2.0-677.gf7870454.el9 ceph 126 k 2026-02-20T21:02:15.288 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:15.288 INFO:teuthology.orchestra.run.trial196.stdout:Transaction Summary 2026-02-20T21:02:15.289 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:15.289 INFO:teuthology.orchestra.run.trial196.stdout:Install 1 Package 2026-02-20T21:02:15.289 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:15.289 INFO:teuthology.orchestra.run.trial196.stdout:Total download size: 126 k 2026-02-20T21:02:15.289 INFO:teuthology.orchestra.run.trial196.stdout:Installed size: 449 k 2026-02-20T21:02:15.289 INFO:teuthology.orchestra.run.trial196.stdout:Downloading Packages: 2026-02-20T21:02:15.654 INFO:teuthology.orchestra.run.trial196.stdout:librados-devel-20.2.0-677.gf7870454.el9.x86_64. 343 kB/s | 126 kB 00:00 2026-02-20T21:02:15.654 INFO:teuthology.orchestra.run.trial196.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:02:15.654 INFO:teuthology.orchestra.run.trial196.stdout:Total 342 kB/s | 126 kB 00:00 2026-02-20T21:02:15.654 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction check 2026-02-20T21:02:15.657 INFO:teuthology.orchestra.run.trial196.stdout:Transaction check succeeded. 2026-02-20T21:02:15.657 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction test 2026-02-20T21:02:15.688 INFO:teuthology.orchestra.run.trial196.stdout:Transaction test succeeded. 2026-02-20T21:02:15.688 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction 2026-02-20T21:02:15.742 INFO:teuthology.orchestra.run.trial196.stdout: Preparing : 1/1 2026-02-20T21:02:15.759 INFO:teuthology.orchestra.run.trial196.stdout: Installing : librados-devel-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:15.837 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: librados-devel-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:15.895 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : librados-devel-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:15.895 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:15.895 INFO:teuthology.orchestra.run.trial196.stdout:Installed: 2026-02-20T21:02:15.895 INFO:teuthology.orchestra.run.trial196.stdout: librados-devel-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:02:15.895 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:15.896 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:02:15.923 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install libcephfs2 2026-02-20T21:02:16.239 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:53 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:02:16.304 INFO:teuthology.orchestra.run.trial196.stdout:Package libcephfs2-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:02:16.322 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:02:16.323 INFO:teuthology.orchestra.run.trial196.stdout:Nothing to do. 2026-02-20T21:02:16.323 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:02:16.344 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install libcephfs-devel 2026-02-20T21:02:16.655 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:53 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:02:16.739 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:02:16.739 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:16.739 INFO:teuthology.orchestra.run.trial196.stdout: Package Arch Version Repo Size 2026-02-20T21:02:16.740 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:16.740 INFO:teuthology.orchestra.run.trial196.stdout:Installing: 2026-02-20T21:02:16.740 INFO:teuthology.orchestra.run.trial196.stdout: libcephfs-devel x86_64 2:20.2.0-677.gf7870454.el9 ceph 34 k 2026-02-20T21:02:16.740 INFO:teuthology.orchestra.run.trial196.stdout:Installing dependencies: 2026-02-20T21:02:16.740 INFO:teuthology.orchestra.run.trial196.stdout: libcephfs-proxy2 x86_64 2:20.2.0-677.gf7870454.el9 ceph 24 k 2026-02-20T21:02:16.740 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:16.740 INFO:teuthology.orchestra.run.trial196.stdout:Transaction Summary 2026-02-20T21:02:16.740 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:16.740 INFO:teuthology.orchestra.run.trial196.stdout:Install 2 Packages 2026-02-20T21:02:16.741 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:16.741 INFO:teuthology.orchestra.run.trial196.stdout:Total download size: 58 k 2026-02-20T21:02:16.741 INFO:teuthology.orchestra.run.trial196.stdout:Installed size: 207 k 2026-02-20T21:02:16.741 INFO:teuthology.orchestra.run.trial196.stdout:Downloading Packages: 2026-02-20T21:02:16.804 INFO:teuthology.orchestra.run.trial196.stdout:(1/2): libcephfs-devel-20.2.0-677.gf7870454.el9 530 kB/s | 34 kB 00:00 2026-02-20T21:02:16.816 INFO:teuthology.orchestra.run.trial196.stdout:(2/2): libcephfs-proxy2-20.2.0-677.gf7870454.el 314 kB/s | 24 kB 00:00 2026-02-20T21:02:16.817 INFO:teuthology.orchestra.run.trial196.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:02:16.817 INFO:teuthology.orchestra.run.trial196.stdout:Total 749 kB/s | 58 kB 00:00 2026-02-20T21:02:16.818 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction check 2026-02-20T21:02:16.820 INFO:teuthology.orchestra.run.trial196.stdout:Transaction check succeeded. 2026-02-20T21:02:16.820 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction test 2026-02-20T21:02:16.851 INFO:teuthology.orchestra.run.trial196.stdout:Transaction test succeeded. 2026-02-20T21:02:16.852 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction 2026-02-20T21:02:16.900 INFO:teuthology.orchestra.run.trial196.stdout: Preparing : 1/1 2026-02-20T21:02:16.901 INFO:teuthology.orchestra.run.trial196.stdout: Installing : libcephfs-proxy2-2:20.2.0-677.gf7870454.el9.x86_64 1/2 2026-02-20T21:02:16.915 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: libcephfs-proxy2-2:20.2.0-677.gf7870454.el9.x86_64 1/2 2026-02-20T21:02:16.931 INFO:teuthology.orchestra.run.trial196.stdout: Installing : libcephfs-devel-2:20.2.0-677.gf7870454.el9.x86_64 2/2 2026-02-20T21:02:16.987 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: libcephfs-devel-2:20.2.0-677.gf7870454.el9.x86_64 2/2 2026-02-20T21:02:16.987 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : libcephfs-devel-2:20.2.0-677.gf7870454.el9.x86_64 1/2 2026-02-20T21:02:17.045 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : libcephfs-proxy2-2:20.2.0-677.gf7870454.el9.x86_64 2/2 2026-02-20T21:02:17.045 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:17.045 INFO:teuthology.orchestra.run.trial196.stdout:Installed: 2026-02-20T21:02:17.045 INFO:teuthology.orchestra.run.trial196.stdout: libcephfs-devel-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:02:17.045 INFO:teuthology.orchestra.run.trial196.stdout: libcephfs-proxy2-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:02:17.045 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:17.045 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:02:17.072 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install librados2 2026-02-20T21:02:17.375 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:54 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:02:17.441 INFO:teuthology.orchestra.run.trial196.stdout:Package librados2-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:02:17.460 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:02:17.460 INFO:teuthology.orchestra.run.trial196.stdout:Nothing to do. 2026-02-20T21:02:17.460 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:02:17.482 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install librbd1 2026-02-20T21:02:17.793 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:54 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:02:17.859 INFO:teuthology.orchestra.run.trial196.stdout:Package librbd1-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:02:17.877 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:02:17.877 INFO:teuthology.orchestra.run.trial196.stdout:Nothing to do. 2026-02-20T21:02:17.877 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:02:17.898 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install python3-rados 2026-02-20T21:02:18.206 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:55 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:02:18.273 INFO:teuthology.orchestra.run.trial196.stdout:Package python3-rados-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:02:18.291 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:02:18.292 INFO:teuthology.orchestra.run.trial196.stdout:Nothing to do. 2026-02-20T21:02:18.292 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:02:18.313 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install python3-rgw 2026-02-20T21:02:18.624 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:55 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:02:18.690 INFO:teuthology.orchestra.run.trial196.stdout:Package python3-rgw-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:02:18.708 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:02:18.709 INFO:teuthology.orchestra.run.trial196.stdout:Nothing to do. 2026-02-20T21:02:18.709 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:02:18.731 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install python3-cephfs 2026-02-20T21:02:19.040 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:56 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:02:19.105 INFO:teuthology.orchestra.run.trial196.stdout:Package python3-cephfs-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:02:19.124 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:02:19.125 INFO:teuthology.orchestra.run.trial196.stdout:Nothing to do. 2026-02-20T21:02:19.125 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:02:19.148 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install python3-rbd 2026-02-20T21:02:19.455 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:56 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:02:19.521 INFO:teuthology.orchestra.run.trial196.stdout:Package python3-rbd-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:02:19.539 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:02:19.540 INFO:teuthology.orchestra.run.trial196.stdout:Nothing to do. 2026-02-20T21:02:19.540 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:02:19.562 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install rbd-fuse 2026-02-20T21:02:19.868 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:56 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:02:19.954 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:02:19.954 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:19.955 INFO:teuthology.orchestra.run.trial196.stdout: Package Architecture Version Repository Size 2026-02-20T21:02:19.955 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:19.955 INFO:teuthology.orchestra.run.trial196.stdout:Installing: 2026-02-20T21:02:19.955 INFO:teuthology.orchestra.run.trial196.stdout: rbd-fuse x86_64 2:20.2.0-677.gf7870454.el9 ceph 91 k 2026-02-20T21:02:19.955 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:19.955 INFO:teuthology.orchestra.run.trial196.stdout:Transaction Summary 2026-02-20T21:02:19.955 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:19.955 INFO:teuthology.orchestra.run.trial196.stdout:Install 1 Package 2026-02-20T21:02:19.955 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:19.955 INFO:teuthology.orchestra.run.trial196.stdout:Total download size: 91 k 2026-02-20T21:02:19.956 INFO:teuthology.orchestra.run.trial196.stdout:Installed size: 238 k 2026-02-20T21:02:19.956 INFO:teuthology.orchestra.run.trial196.stdout:Downloading Packages: 2026-02-20T21:02:20.030 INFO:teuthology.orchestra.run.trial196.stdout:rbd-fuse-20.2.0-677.gf7870454.el9.x86_64.rpm 1.2 MB/s | 91 kB 00:00 2026-02-20T21:02:20.030 INFO:teuthology.orchestra.run.trial196.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:02:20.031 INFO:teuthology.orchestra.run.trial196.stdout:Total 1.2 MB/s | 91 kB 00:00 2026-02-20T21:02:20.031 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction check 2026-02-20T21:02:20.035 INFO:teuthology.orchestra.run.trial196.stdout:Transaction check succeeded. 2026-02-20T21:02:20.036 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction test 2026-02-20T21:02:20.069 INFO:teuthology.orchestra.run.trial196.stdout:Transaction test succeeded. 2026-02-20T21:02:20.069 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction 2026-02-20T21:02:20.120 INFO:teuthology.orchestra.run.trial196.stdout: Preparing : 1/1 2026-02-20T21:02:20.137 INFO:teuthology.orchestra.run.trial196.stdout: Installing : rbd-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:20.219 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: rbd-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:20.278 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : rbd-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:20.278 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:20.278 INFO:teuthology.orchestra.run.trial196.stdout:Installed: 2026-02-20T21:02:20.278 INFO:teuthology.orchestra.run.trial196.stdout: rbd-fuse-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:02:20.278 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:20.278 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:02:20.307 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install rbd-mirror 2026-02-20T21:02:20.615 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:57 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:02:20.699 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:02:20.699 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:20.699 INFO:teuthology.orchestra.run.trial196.stdout: Package Arch Version Repo Size 2026-02-20T21:02:20.700 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:20.700 INFO:teuthology.orchestra.run.trial196.stdout:Installing: 2026-02-20T21:02:20.700 INFO:teuthology.orchestra.run.trial196.stdout: rbd-mirror x86_64 2:20.2.0-677.gf7870454.el9 ceph 2.9 M 2026-02-20T21:02:20.700 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:20.700 INFO:teuthology.orchestra.run.trial196.stdout:Transaction Summary 2026-02-20T21:02:20.700 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:20.700 INFO:teuthology.orchestra.run.trial196.stdout:Install 1 Package 2026-02-20T21:02:20.700 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:20.700 INFO:teuthology.orchestra.run.trial196.stdout:Total download size: 2.9 M 2026-02-20T21:02:20.700 INFO:teuthology.orchestra.run.trial196.stdout:Installed size: 11 M 2026-02-20T21:02:20.700 INFO:teuthology.orchestra.run.trial196.stdout:Downloading Packages: 2026-02-20T21:02:20.867 INFO:teuthology.orchestra.run.trial196.stdout:rbd-mirror-20.2.0-677.gf7870454.el9.x86_64.rpm 17 MB/s | 2.9 MB 00:00 2026-02-20T21:02:20.868 INFO:teuthology.orchestra.run.trial196.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:02:20.868 INFO:teuthology.orchestra.run.trial196.stdout:Total 17 MB/s | 2.9 MB 00:00 2026-02-20T21:02:20.868 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction check 2026-02-20T21:02:20.873 INFO:teuthology.orchestra.run.trial196.stdout:Transaction check succeeded. 2026-02-20T21:02:20.873 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction test 2026-02-20T21:02:20.911 INFO:teuthology.orchestra.run.trial196.stdout:Transaction test succeeded. 2026-02-20T21:02:20.911 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction 2026-02-20T21:02:21.093 INFO:teuthology.orchestra.run.trial196.stdout: Preparing : 1/1 2026-02-20T21:02:21.094 INFO:teuthology.orchestra.run.trial196.stdout: Installing : rbd-mirror-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:21.118 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: rbd-mirror-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:21.118 INFO:teuthology.orchestra.run.trial196.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:02:21.118 INFO:teuthology.orchestra.run.trial196.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-02-20T21:02:21.118 INFO:teuthology.orchestra.run.trial196.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-20T21:02:21.118 INFO:teuthology.orchestra.run.trial196.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-20T21:02:21.118 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:21.429 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : rbd-mirror-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:21.429 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:21.429 INFO:teuthology.orchestra.run.trial196.stdout:Installed: 2026-02-20T21:02:21.429 INFO:teuthology.orchestra.run.trial196.stdout: rbd-mirror-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:02:21.429 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:21.429 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:02:21.459 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install rbd-nbd 2026-02-20T21:02:21.772 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:58 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:02:21.856 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:02:21.856 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:21.857 INFO:teuthology.orchestra.run.trial196.stdout: Package Architecture Version Repository Size 2026-02-20T21:02:21.857 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:21.857 INFO:teuthology.orchestra.run.trial196.stdout:Installing: 2026-02-20T21:02:21.857 INFO:teuthology.orchestra.run.trial196.stdout: rbd-nbd x86_64 2:20.2.0-677.gf7870454.el9 ceph 180 k 2026-02-20T21:02:21.857 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:21.857 INFO:teuthology.orchestra.run.trial196.stdout:Transaction Summary 2026-02-20T21:02:21.857 INFO:teuthology.orchestra.run.trial196.stdout:================================================================================ 2026-02-20T21:02:21.857 INFO:teuthology.orchestra.run.trial196.stdout:Install 1 Package 2026-02-20T21:02:21.857 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:21.857 INFO:teuthology.orchestra.run.trial196.stdout:Total download size: 180 k 2026-02-20T21:02:21.858 INFO:teuthology.orchestra.run.trial196.stdout:Installed size: 498 k 2026-02-20T21:02:21.858 INFO:teuthology.orchestra.run.trial196.stdout:Downloading Packages: 2026-02-20T21:02:21.970 INFO:teuthology.orchestra.run.trial196.stdout:rbd-nbd-20.2.0-677.gf7870454.el9.x86_64.rpm 1.5 MB/s | 180 kB 00:00 2026-02-20T21:02:21.970 INFO:teuthology.orchestra.run.trial196.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:02:21.971 INFO:teuthology.orchestra.run.trial196.stdout:Total 1.5 MB/s | 180 kB 00:00 2026-02-20T21:02:21.971 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction check 2026-02-20T21:02:21.975 INFO:teuthology.orchestra.run.trial196.stdout:Transaction check succeeded. 2026-02-20T21:02:21.975 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction test 2026-02-20T21:02:22.007 INFO:teuthology.orchestra.run.trial196.stdout:Transaction test succeeded. 2026-02-20T21:02:22.007 INFO:teuthology.orchestra.run.trial196.stdout:Running transaction 2026-02-20T21:02:22.063 INFO:teuthology.orchestra.run.trial196.stdout: Preparing : 1/1 2026-02-20T21:02:22.080 INFO:teuthology.orchestra.run.trial196.stdout: Installing : rbd-nbd-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:22.159 INFO:teuthology.orchestra.run.trial196.stdout: Running scriptlet: rbd-nbd-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:22.216 INFO:teuthology.orchestra.run.trial196.stdout: Verifying : rbd-nbd-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:02:22.216 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:22.216 INFO:teuthology.orchestra.run.trial196.stdout:Installed: 2026-02-20T21:02:22.216 INFO:teuthology.orchestra.run.trial196.stdout: rbd-nbd-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:02:22.217 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:22.217 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:02:22.245 DEBUG:teuthology.orchestra.run.trial196:> sudo yum -y install rbd-mirror 2026-02-20T21:02:22.557 INFO:teuthology.orchestra.run.trial196.stdout:Last metadata expiration check: 0:00:59 ago on Fri 20 Feb 2026 09:01:23 PM UTC. 2026-02-20T21:02:22.622 INFO:teuthology.orchestra.run.trial196.stdout:Package rbd-mirror-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:02:22.641 INFO:teuthology.orchestra.run.trial196.stdout:Dependencies resolved. 2026-02-20T21:02:22.642 INFO:teuthology.orchestra.run.trial196.stdout:Nothing to do. 2026-02-20T21:02:22.642 INFO:teuthology.orchestra.run.trial196.stdout:Complete! 2026-02-20T21:02:22.666 DEBUG:teuthology.parallel:result is None 2026-02-20T21:02:22.667 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f7870454149f22574086dd2ff55d050585136531 2026-02-20T21:02:22.801 DEBUG:teuthology.orchestra.run.trial196:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2026-02-20T21:02:22.822 INFO:teuthology.orchestra.run.trial196.stdout:20.2.0-677.gf7870454.el9 2026-02-20T21:02:22.822 INFO:teuthology.packaging:The installed version of ceph is 20.2.0-677.gf7870454.el9 2026-02-20T21:02:22.822 INFO:teuthology.task.install:The correct ceph version 20.2.0-677.gf7870454 is installed. 2026-02-20T21:02:22.824 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f7870454149f22574086dd2ff55d050585136531 2026-02-20T21:02:22.962 DEBUG:teuthology.orchestra.run.trial199:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2026-02-20T21:02:22.983 INFO:teuthology.orchestra.run.trial199.stdout:20.2.0-677.gf7870454.el9 2026-02-20T21:02:22.983 INFO:teuthology.packaging:The installed version of ceph is 20.2.0-677.gf7870454.el9 2026-02-20T21:02:22.983 INFO:teuthology.task.install:The correct ceph version 20.2.0-677.gf7870454 is installed. 2026-02-20T21:02:22.985 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2026-02-20T21:02:22.985 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T21:02:22.985 DEBUG:teuthology.orchestra.run.trial196:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-20T21:02:23.011 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:23.011 DEBUG:teuthology.orchestra.run.trial199:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-20T21:02:23.049 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2026-02-20T21:02:23.049 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T21:02:23.049 DEBUG:teuthology.orchestra.run.trial196:> sudo dd of=/usr/bin/daemon-helper 2026-02-20T21:02:23.075 DEBUG:teuthology.orchestra.run.trial196:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-20T21:02:23.141 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:23.141 DEBUG:teuthology.orchestra.run.trial199:> sudo dd of=/usr/bin/daemon-helper 2026-02-20T21:02:23.168 DEBUG:teuthology.orchestra.run.trial199:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-20T21:02:23.233 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2026-02-20T21:02:23.234 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T21:02:23.234 DEBUG:teuthology.orchestra.run.trial196:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-20T21:02:23.259 DEBUG:teuthology.orchestra.run.trial196:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-20T21:02:23.324 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:23.324 DEBUG:teuthology.orchestra.run.trial199:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-20T21:02:23.349 DEBUG:teuthology.orchestra.run.trial199:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-20T21:02:23.412 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2026-02-20T21:02:23.413 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T21:02:23.413 DEBUG:teuthology.orchestra.run.trial196:> sudo dd of=/usr/bin/stdin-killer 2026-02-20T21:02:23.439 DEBUG:teuthology.orchestra.run.trial196:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-20T21:02:23.504 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:23.504 DEBUG:teuthology.orchestra.run.trial199:> sudo dd of=/usr/bin/stdin-killer 2026-02-20T21:02:23.529 DEBUG:teuthology.orchestra.run.trial199:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-20T21:02:23.593 INFO:teuthology.run_tasks:Running task ceph... 2026-02-20T21:02:23.709 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2026-02-20T21:02:23.709 DEBUG:teuthology.orchestra.run.trial196:> sudo chmod 777 /var/log/ceph 2026-02-20T21:02:23.711 DEBUG:teuthology.orchestra.run.trial199:> sudo chmod 777 /var/log/ceph 2026-02-20T21:02:23.736 INFO:tasks.ceph:Disabling ceph logrotate... 2026-02-20T21:02:23.736 DEBUG:teuthology.orchestra.run.trial196:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-20T21:02:23.776 DEBUG:teuthology.orchestra.run.trial199:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-20T21:02:23.801 INFO:tasks.ceph:Creating extra log directories... 2026-02-20T21:02:23.801 DEBUG:teuthology.orchestra.run.trial196:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-20T21:02:23.840 DEBUG:teuthology.orchestra.run.trial199:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-20T21:02:23.884 INFO:tasks.ceph:Creating ceph cluster cluster1... 2026-02-20T21:02:23.884 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'snappy', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-20T21:02:23.884 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61415', 'branch': 'wip-pdonnell-testing-20260219.182737-tentacle', 'description': 'rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-snappy supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-workunit-fast-diff}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '61415', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run the rbd_mirror.sh workunit to test the rbd-mirror daemon in snapshot mode'}], 'name': 'yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial', 'no_nested_subset': False, 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'wip-pdonnell-testing-20260219.182737-tentacle'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'snappy', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-pdonnell-testing-20260219.182737-tentacle', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 7161, 'sha1': 'f7870454149f22574086dd2ff55d050585136531', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': 'wip-pdonnell-testing-20260219.182737-tentacle', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_f7870454149f22574086dd2ff55d050585136531/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'f7870454149f22574086dd2ff55d050585136531', 'targets': {'trial196.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNXSz/oleXj4WCyNHwLu/9imobgWwskZJuDoZU2v2KihsuN8A40ibPZsGroPVka9yfVxYbjvvfMQvHwms8hWr/I=', 'trial199.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPM+ZdBRamPjFyytTKVlvfP6NMc801M2vKxXYLe18YXGDgXMyw4c98dOBWD/E0Bn+Cp7UqFHFWoxeALuafPFuco='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'snappy', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, {'ceph': {'cluster': 'cluster2'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror.sh']}, 'env': {'CEPH_ARGS': '', 'RBD_IMAGE_FEATURES': 'layering,exclusive-lock,object-map,fast-diff', 'RBD_MIRROR_CONFIG_KEY': '1', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1'}}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-20_16:15:01', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.1540699'} 2026-02-20T21:02:23.885 DEBUG:teuthology.orchestra.run.trial196:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-20T21:02:23.904 DEBUG:teuthology.orchestra.run.trial199:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-20T21:02:23.922 DEBUG:teuthology.orchestra.run.trial196:> sudo install -d -m0777 -- /var/run/ceph 2026-02-20T21:02:23.961 DEBUG:teuthology.orchestra.run.trial199:> sudo install -d -m0777 -- /var/run/ceph 2026-02-20T21:02:23.986 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T21:02:23.986 DEBUG:teuthology.orchestra.run.trial196:> dd if=/scratch_devs of=/dev/stdout 2026-02-20T21:02:24.041 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-20T21:02:24.041 DEBUG:teuthology.orchestra.run.trial196:> stat /dev/vg_nvme/lv_1 2026-02-20T21:02:24.098 INFO:teuthology.orchestra.run.trial196.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-20T21:02:24.098 INFO:teuthology.orchestra.run.trial196.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:02:24.098 INFO:teuthology.orchestra.run.trial196.stdout:Device: 6h/6d Inode: 917 Links: 1 2026-02-20T21:02:24.098 INFO:teuthology.orchestra.run.trial196.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:02:24.098 INFO:teuthology.orchestra.run.trial196.stdout:Context: system_u:object_r:device_t:s0 2026-02-20T21:02:24.098 INFO:teuthology.orchestra.run.trial196.stdout:Access: 2026-02-20 21:02:21.255911867 +0000 2026-02-20T21:02:24.099 INFO:teuthology.orchestra.run.trial196.stdout:Modify: 2026-02-20 21:00:51.278105279 +0000 2026-02-20T21:02:24.099 INFO:teuthology.orchestra.run.trial196.stdout:Change: 2026-02-20 21:00:51.278105279 +0000 2026-02-20T21:02:24.099 INFO:teuthology.orchestra.run.trial196.stdout: Birth: 2026-02-20 21:00:51.278105279 +0000 2026-02-20T21:02:24.099 DEBUG:teuthology.orchestra.run.trial196:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-20T21:02:24.160 INFO:teuthology.orchestra.run.trial196.stderr:1+0 records in 2026-02-20T21:02:24.160 INFO:teuthology.orchestra.run.trial196.stderr:1+0 records out 2026-02-20T21:02:24.160 INFO:teuthology.orchestra.run.trial196.stderr:512 bytes copied, 0.000151732 s, 3.4 MB/s 2026-02-20T21:02:24.161 DEBUG:teuthology.orchestra.run.trial196:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-20T21:02:24.219 DEBUG:teuthology.orchestra.run.trial196:> stat /dev/vg_nvme/lv_2 2026-02-20T21:02:24.274 INFO:teuthology.orchestra.run.trial196.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-20T21:02:24.274 INFO:teuthology.orchestra.run.trial196.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:02:24.274 INFO:teuthology.orchestra.run.trial196.stdout:Device: 6h/6d Inode: 943 Links: 1 2026-02-20T21:02:24.274 INFO:teuthology.orchestra.run.trial196.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:02:24.274 INFO:teuthology.orchestra.run.trial196.stdout:Context: system_u:object_r:device_t:s0 2026-02-20T21:02:24.274 INFO:teuthology.orchestra.run.trial196.stdout:Access: 2026-02-20 21:02:21.255911867 +0000 2026-02-20T21:02:24.274 INFO:teuthology.orchestra.run.trial196.stdout:Modify: 2026-02-20 21:00:51.512100107 +0000 2026-02-20T21:02:24.275 INFO:teuthology.orchestra.run.trial196.stdout:Change: 2026-02-20 21:00:51.512100107 +0000 2026-02-20T21:02:24.275 INFO:teuthology.orchestra.run.trial196.stdout: Birth: 2026-02-20 21:00:51.512100107 +0000 2026-02-20T21:02:24.275 DEBUG:teuthology.orchestra.run.trial196:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-20T21:02:24.337 INFO:teuthology.orchestra.run.trial196.stderr:1+0 records in 2026-02-20T21:02:24.337 INFO:teuthology.orchestra.run.trial196.stderr:1+0 records out 2026-02-20T21:02:24.337 INFO:teuthology.orchestra.run.trial196.stderr:512 bytes copied, 0.000149097 s, 3.4 MB/s 2026-02-20T21:02:24.338 DEBUG:teuthology.orchestra.run.trial196:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-20T21:02:24.396 DEBUG:teuthology.orchestra.run.trial196:> stat /dev/vg_nvme/lv_3 2026-02-20T21:02:24.453 INFO:teuthology.orchestra.run.trial196.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-20T21:02:24.454 INFO:teuthology.orchestra.run.trial196.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:02:24.454 INFO:teuthology.orchestra.run.trial196.stdout:Device: 6h/6d Inode: 971 Links: 1 2026-02-20T21:02:24.454 INFO:teuthology.orchestra.run.trial196.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:02:24.454 INFO:teuthology.orchestra.run.trial196.stdout:Context: system_u:object_r:device_t:s0 2026-02-20T21:02:24.454 INFO:teuthology.orchestra.run.trial196.stdout:Access: 2026-02-20 21:02:21.255911867 +0000 2026-02-20T21:02:24.454 INFO:teuthology.orchestra.run.trial196.stdout:Modify: 2026-02-20 21:00:51.746094936 +0000 2026-02-20T21:02:24.454 INFO:teuthology.orchestra.run.trial196.stdout:Change: 2026-02-20 21:00:51.746094936 +0000 2026-02-20T21:02:24.454 INFO:teuthology.orchestra.run.trial196.stdout: Birth: 2026-02-20 21:00:51.746094936 +0000 2026-02-20T21:02:24.455 DEBUG:teuthology.orchestra.run.trial196:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-20T21:02:24.517 INFO:teuthology.orchestra.run.trial196.stderr:1+0 records in 2026-02-20T21:02:24.517 INFO:teuthology.orchestra.run.trial196.stderr:1+0 records out 2026-02-20T21:02:24.517 INFO:teuthology.orchestra.run.trial196.stderr:512 bytes copied, 0.000147133 s, 3.5 MB/s 2026-02-20T21:02:24.519 DEBUG:teuthology.orchestra.run.trial196:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-20T21:02:24.575 DEBUG:teuthology.orchestra.run.trial196:> stat /dev/vg_nvme/lv_4 2026-02-20T21:02:24.632 INFO:teuthology.orchestra.run.trial196.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-20T21:02:24.632 INFO:teuthology.orchestra.run.trial196.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:02:24.632 INFO:teuthology.orchestra.run.trial196.stdout:Device: 6h/6d Inode: 999 Links: 1 2026-02-20T21:02:24.632 INFO:teuthology.orchestra.run.trial196.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:02:24.632 INFO:teuthology.orchestra.run.trial196.stdout:Context: system_u:object_r:device_t:s0 2026-02-20T21:02:24.632 INFO:teuthology.orchestra.run.trial196.stdout:Access: 2026-02-20 21:02:21.256911848 +0000 2026-02-20T21:02:24.633 INFO:teuthology.orchestra.run.trial196.stdout:Modify: 2026-02-20 21:00:52.015088990 +0000 2026-02-20T21:02:24.633 INFO:teuthology.orchestra.run.trial196.stdout:Change: 2026-02-20 21:00:52.015088990 +0000 2026-02-20T21:02:24.633 INFO:teuthology.orchestra.run.trial196.stdout: Birth: 2026-02-20 21:00:52.015088990 +0000 2026-02-20T21:02:24.633 DEBUG:teuthology.orchestra.run.trial196:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-20T21:02:24.695 INFO:teuthology.orchestra.run.trial196.stderr:1+0 records in 2026-02-20T21:02:24.695 INFO:teuthology.orchestra.run.trial196.stderr:1+0 records out 2026-02-20T21:02:24.695 INFO:teuthology.orchestra.run.trial196.stderr:512 bytes copied, 0.000143176 s, 3.6 MB/s 2026-02-20T21:02:24.697 DEBUG:teuthology.orchestra.run.trial196:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-20T21:02:24.753 INFO:tasks.ceph:osd dev map: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:02:24.753 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial196.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-20T21:02:24.753 INFO:tasks.ceph:Generating config... 2026-02-20T21:02:24.754 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-20T21:02:24.754 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-20T21:02:24.754 INFO:tasks.ceph:[client] log to stderr = False 2026-02-20T21:02:24.754 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-20T21:02:24.755 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:24.755 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:24.755 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:24.755 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:24.755 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:24.755 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:24.755 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:24.755 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:24.755 INFO:tasks.ceph:[client.mirror.4] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:24.755 INFO:tasks.ceph:[client.mirror.4] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:24.755 INFO:tasks.ceph:[client.mirror.5] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:24.755 INFO:tasks.ceph:[client.mirror.5] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:24.755 INFO:tasks.ceph:[client.mirror.6] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:24.755 INFO:tasks.ceph:[client.mirror.6] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:24.755 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-20T21:02:24.755 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-20T21:02:24.755 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-20T21:02:24.755 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-20T21:02:24.755 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-20T21:02:24.755 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-20T21:02:24.755 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-20T21:02:24.755 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-20T21:02:24.755 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-20T21:02:24.755 INFO:tasks.ceph:[osd] bluestore compression algorithm = snappy 2026-02-20T21:02:24.755 INFO:tasks.ceph:[osd] bluestore compression mode = aggressive 2026-02-20T21:02:24.756 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-20T21:02:24.756 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-20T21:02:24.756 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-20T21:02:24.756 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-20T21:02:24.756 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-20T21:02:24.756 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-20T21:02:24.756 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-20T21:02:24.756 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-20T21:02:24.756 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-20T21:02:24.756 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-20T21:02:24.756 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-20T21:02:24.756 INFO:tasks.ceph:Setting up cluster1.mon.a... 2026-02-20T21:02:24.756 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster1.keyring 2026-02-20T21:02:24.829 INFO:teuthology.orchestra.run.trial196.stdout:creating /etc/ceph/cluster1.keyring 2026-02-20T21:02:24.831 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster1.keyring 2026-02-20T21:02:24.909 DEBUG:teuthology.orchestra.run.trial196:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-20T21:02:24.973 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster1.mon.a', '10.20.193.196')] 2026-02-20T21:02:24.974 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.196', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluestore block size': 96636764160, 'bluestore compression algorithm': 'snappy', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-20T21:02:24.975 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T21:02:24.976 DEBUG:teuthology.orchestra.run.trial196:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-20T21:02:25.031 DEBUG:teuthology.orchestra.run.trial196:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.196 --print /home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:02:25.102 INFO:teuthology.orchestra.run.trial196.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:02:25.103 INFO:teuthology.orchestra.run.trial196.stdout:monmaptool: generated fsid d723a433-cea3-4643-ada0-27ddcd59cc6f 2026-02-20T21:02:25.103 INFO:teuthology.orchestra.run.trial196.stdout:setting min_mon_release = tentacle 2026-02-20T21:02:25.103 INFO:teuthology.orchestra.run.trial196.stdout:epoch 0 2026-02-20T21:02:25.103 INFO:teuthology.orchestra.run.trial196.stdout:fsid d723a433-cea3-4643-ada0-27ddcd59cc6f 2026-02-20T21:02:25.103 INFO:teuthology.orchestra.run.trial196.stdout:last_changed 2026-02-20T21:02:25.101051+0000 2026-02-20T21:02:25.103 INFO:teuthology.orchestra.run.trial196.stdout:created 2026-02-20T21:02:25.101051+0000 2026-02-20T21:02:25.103 INFO:teuthology.orchestra.run.trial196.stdout:min_mon_release 20 (tentacle) 2026-02-20T21:02:25.103 INFO:teuthology.orchestra.run.trial196.stdout:election_strategy: 1 2026-02-20T21:02:25.103 INFO:teuthology.orchestra.run.trial196.stdout:0: [v2:10.20.193.196:3300/0,v1:10.20.193.196:6789/0] mon.a 2026-02-20T21:02:25.104 INFO:teuthology.orchestra.run.trial196.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster1.monmap (1 monitors) 2026-02-20T21:02:25.105 DEBUG:teuthology.orchestra.run.trial196:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-20T21:02:25.161 INFO:tasks.ceph:Writing /etc/ceph/cluster1.conf for FSID d723a433-cea3-4643-ada0-27ddcd59cc6f... 2026-02-20T21:02:25.162 DEBUG:teuthology.orchestra.run.trial196:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-20T21:02:25.203 DEBUG:teuthology.orchestra.run.trial199:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-20T21:02:25.241 INFO:teuthology.orchestra.run.trial196.stdout:[global] 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: chdir = "" 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: auth supported = cephx 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: filestore xattr use omap = true 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: mon clock drift allowed = 1.000 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: osd crush chooseleaf type = 0 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: auth debug = true 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: ms die on old message = true 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: ms die on bug = true 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: mon pg warn max object skew = 0 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: # disable pg_autoscaler by default for new pools 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: osd pool default size = 2 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: mon osd allow primary affinity = true 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: mon osd allow pg remap = true 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: mon warn on legacy crush tunables = false 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: mon warn on crush straw calc version zero = false 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: mon warn on no sortbitwise = false 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: mon warn on osd down out interval zero = false 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: mon warn on too few osds = false 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: mon_allow_pool_size_one = true 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: osd default data pool replay window = 5 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: mon allow pool delete = true 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: mon cluster log file level = debug 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: debug asserts on shutdown = true 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: mon health detail to clog = false 2026-02-20T21:02:25.242 INFO:teuthology.orchestra.run.trial196.stdout: mon host = 10.20.193.196 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: mon client directed command retry = 5 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: mon warn on pool no app = False 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: ms inject socket failures = 5000 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: fsid = d723a433-cea3-4643-ada0-27ddcd59cc6f 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout:[osd] 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: osd journal size = 100 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: osd scrub load threshold = 5.0 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: osd scrub max interval = 600 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: osd mclock profile = high_recovery_ops 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: osd mclock skip benchmark = true 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: osd recover clone overlap = true 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: osd recovery max chunk = 1048576 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: osd debug shutdown = true 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: osd debug op order = true 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: osd debug verify stray on activate = true 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: osd debug trim objects = true 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: osd open classes on start = true 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: osd debug pg log writeout = true 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: osd deep scrub update digest min age = 30 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: osd map max advance = 10 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: journal zero on create = true 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: filestore ondisk finisher threads = 3 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: filestore apply finisher threads = 3 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: bdev debug aio = true 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: osd debug misdirected ops = true 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: bluestore block size = 96636764160 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: bluestore compression algorithm = snappy 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: bluestore compression mode = aggressive 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: bluestore fsck on mount = True 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: debug bluefs = 1/20 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: debug bluestore = 1/20 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: debug ms = 1 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: debug osd = 20 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: debug rocksdb = 4/10 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: mon osd backfillfull_ratio = 0.85 2026-02-20T21:02:25.243 INFO:teuthology.orchestra.run.trial196.stdout: mon osd full ratio = 0.9 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: mon osd nearfull ratio = 0.8 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: osd failsafe full ratio = 0.95 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: osd objectstore = bluestore 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout:[mgr] 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: debug ms = 1 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: debug mgr = 20 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: debug mon = 20 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: debug auth = 20 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: mgr/telemetry/nag = false 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout:[mon] 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: debug ms = 1 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: debug mon = 20 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: debug paxos = 20 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: debug auth = 20 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: mon data avail warn = 5 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: mon mgr mkfs grace = 240 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: mon osd reporter subtree level = osd 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: mon osd prime pg temp = true 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: auth mon ticket ttl = 660 # 11m 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: auth service ticket ttl = 240 # 4m 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: # don't complain about insecure global_id in the test suite 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: # 1m isn't quite enough 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: mon_down_mkfs_grace = 2m 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: mon_warn_on_filestore_osds = false 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout:[client] 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: rgw cache enabled = true 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: rgw enable ops log = true 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: rgw enable usage log = true 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: debug rbd = 20 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: debug rbd_mirror = 15 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: log to stderr = False 2026-02-20T21:02:25.244 INFO:teuthology.orchestra.run.trial196.stdout: rbd default features = 125 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial196.stdout:[mon.a] 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial196.stdout:[client.mirror.0] 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial196.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial196.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial196.stdout:[client.mirror.1] 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial196.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial196.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial196.stdout:[client.mirror.2] 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial196.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial196.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial196.stdout:[client.mirror.3] 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial196.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial196.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial196.stdout:[client.mirror.4] 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial196.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial196.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial196.stdout:[client.mirror.5] 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial196.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial196.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial196.stdout:[client.mirror.6] 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial196.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial196.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial199.stdout:[global] 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial199.stdout: chdir = "" 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial199.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial199.stdout: auth supported = cephx 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial199.stdout: filestore xattr use omap = true 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial199.stdout: mon clock drift allowed = 1.000 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial199.stdout: osd crush chooseleaf type = 0 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial199.stdout: auth debug = true 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial199.stdout: ms die on old message = true 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial199.stdout: ms die on bug = true 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.245 INFO:teuthology.orchestra.run.trial199.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: mon pg warn max object skew = 0 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: # disable pg_autoscaler by default for new pools 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: osd pool default size = 2 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: mon osd allow primary affinity = true 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: mon osd allow pg remap = true 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: mon warn on legacy crush tunables = false 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: mon warn on crush straw calc version zero = false 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: mon warn on no sortbitwise = false 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: mon warn on osd down out interval zero = false 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: mon warn on too few osds = false 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: mon_allow_pool_size_one = true 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: osd default data pool replay window = 5 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: mon allow pool delete = true 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: mon cluster log file level = debug 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: debug asserts on shutdown = true 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: mon health detail to clog = false 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: mon host = 10.20.193.196 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: mon client directed command retry = 5 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: mon warn on pool no app = False 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: ms inject socket failures = 5000 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: fsid = d723a433-cea3-4643-ada0-27ddcd59cc6f 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout:[osd] 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: osd journal size = 100 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: osd scrub load threshold = 5.0 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: osd scrub max interval = 600 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: osd mclock profile = high_recovery_ops 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: osd mclock skip benchmark = true 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: osd recover clone overlap = true 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: osd recovery max chunk = 1048576 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: osd debug shutdown = true 2026-02-20T21:02:25.246 INFO:teuthology.orchestra.run.trial199.stdout: osd debug op order = true 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: osd debug verify stray on activate = true 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: osd debug trim objects = true 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: osd open classes on start = true 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: osd debug pg log writeout = true 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: osd deep scrub update digest min age = 30 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: osd map max advance = 10 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: journal zero on create = true 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: filestore ondisk finisher threads = 3 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: filestore apply finisher threads = 3 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: bdev debug aio = true 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: osd debug misdirected ops = true 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: bluestore block size = 96636764160 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: bluestore compression algorithm = snappy 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: bluestore compression mode = aggressive 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: bluestore fsck on mount = True 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: debug bluefs = 1/20 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: debug bluestore = 1/20 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: debug ms = 1 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: debug osd = 20 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: debug rocksdb = 4/10 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: mon osd backfillfull_ratio = 0.85 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: mon osd full ratio = 0.9 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: mon osd nearfull ratio = 0.8 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: osd failsafe full ratio = 0.95 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: osd objectstore = bluestore 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout:[mgr] 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: debug ms = 1 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: debug mgr = 20 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: debug mon = 20 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: debug auth = 20 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: mgr/telemetry/nag = false 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout:[mon] 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: debug ms = 1 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: debug mon = 20 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: debug paxos = 20 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: debug auth = 20 2026-02-20T21:02:25.247 INFO:teuthology.orchestra.run.trial199.stdout: mon data avail warn = 5 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: mon mgr mkfs grace = 240 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: mon osd reporter subtree level = osd 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: mon osd prime pg temp = true 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: auth mon ticket ttl = 660 # 11m 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: auth service ticket ttl = 240 # 4m 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: # don't complain about insecure global_id in the test suite 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: # 1m isn't quite enough 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: mon_down_mkfs_grace = 2m 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: mon_warn_on_filestore_osds = false 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout:[client] 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: rgw cache enabled = true 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: rgw enable ops log = true 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: rgw enable usage log = true 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: debug rbd = 20 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: debug rbd_mirror = 15 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: log to stderr = False 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: rbd default features = 125 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout:[mon.a] 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.0] 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.1] 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.2] 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.3] 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.4] 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.5] 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:25.248 INFO:teuthology.orchestra.run.trial199.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:25.249 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.6] 2026-02-20T21:02:25.249 INFO:teuthology.orchestra.run.trial199.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:25.249 INFO:teuthology.orchestra.run.trial199.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:25.254 INFO:tasks.ceph:Creating admin key on cluster1.mon.a... 2026-02-20T21:02:25.254 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster1.keyring 2026-02-20T21:02:25.334 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-20T21:02:25.334 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T21:02:25.335 DEBUG:teuthology.orchestra.run.trial196:> dd if=/etc/ceph/cluster1.keyring of=/dev/stdout 2026-02-20T21:02:25.349 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T21:02:25.349 DEBUG:teuthology.orchestra.run.trial196:> dd if=/home/ubuntu/cephtest/cluster1.monmap of=/dev/stdout 2026-02-20T21:02:25.407 INFO:tasks.ceph:Sending monmap to node ubuntu@trial196.front.sepia.ceph.com 2026-02-20T21:02:25.407 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T21:02:25.407 DEBUG:teuthology.orchestra.run.trial196:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-20T21:02:25.407 DEBUG:teuthology.orchestra.run.trial196:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-20T21:02:25.481 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T21:02:25.481 DEBUG:teuthology.orchestra.run.trial196:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:02:25.540 INFO:tasks.ceph:Sending monmap to node ubuntu@trial199.front.sepia.ceph.com 2026-02-20T21:02:25.540 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:25.540 DEBUG:teuthology.orchestra.run.trial199:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-20T21:02:25.540 DEBUG:teuthology.orchestra.run.trial199:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-20T21:02:25.574 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:25.575 DEBUG:teuthology.orchestra.run.trial199:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:02:25.630 INFO:tasks.ceph:Setting up mon nodes... 2026-02-20T21:02:25.631 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-20T21:02:25.631 DEBUG:teuthology.orchestra.run.trial196:> sudo mkdir -p /var/lib/ceph/mgr/cluster1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-20T21:02:25.676 INFO:teuthology.orchestra.run.trial196.stdout:creating /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-20T21:02:25.678 INFO:tasks.ceph:Setting up mds nodes... 2026-02-20T21:02:25.679 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-20T21:02:25.679 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster1.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.0.keyring 2026-02-20T21:02:25.712 INFO:teuthology.orchestra.run.trial196.stdout:creating /etc/ceph/cluster1.client.0.keyring 2026-02-20T21:02:25.724 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster1.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.keyring 2026-02-20T21:02:25.761 INFO:teuthology.orchestra.run.trial199.stdout:creating /etc/ceph/cluster1.client.mirror.keyring 2026-02-20T21:02:25.772 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster1.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-20T21:02:25.847 INFO:teuthology.orchestra.run.trial199.stdout:creating /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-20T21:02:25.858 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster1.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-20T21:02:25.934 INFO:teuthology.orchestra.run.trial199.stdout:creating /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-20T21:02:25.945 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster1.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-20T21:02:26.019 INFO:teuthology.orchestra.run.trial199.stdout:creating /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-20T21:02:26.030 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster1.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-20T21:02:26.107 INFO:teuthology.orchestra.run.trial199.stdout:creating /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-20T21:02:26.118 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster1.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-20T21:02:26.193 INFO:teuthology.orchestra.run.trial199.stdout:creating /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-20T21:02:26.204 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster1.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-20T21:02:26.279 INFO:teuthology.orchestra.run.trial199.stdout:creating /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-20T21:02:26.291 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster1.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-20T21:02:26.366 INFO:teuthology.orchestra.run.trial199.stdout:creating /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-20T21:02:26.377 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-20T21:02:26.377 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial196.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-20T21:02:26.377 DEBUG:teuthology.orchestra.run.trial196:> sudo mkdir -p /var/lib/ceph/osd/cluster1-0 2026-02-20T21:02:26.403 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:02:26.404 INFO:tasks.ceph:role: cluster1.osd.0 2026-02-20T21:02:26.404 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial196.front.sepia.ceph.com 2026-02-20T21:02:26.404 DEBUG:teuthology.orchestra.run.trial196:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-20T21:02:26.468 INFO:teuthology.orchestra.run.trial196.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-20T21:02:26.469 INFO:teuthology.orchestra.run.trial196.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-20T21:02:26.469 INFO:teuthology.orchestra.run.trial196.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-20T21:02:26.469 INFO:teuthology.orchestra.run.trial196.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-20T21:02:26.469 INFO:teuthology.orchestra.run.trial196.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-20T21:02:26.469 INFO:teuthology.orchestra.run.trial196.stdout: = sunit=0 swidth=0 blks 2026-02-20T21:02:26.469 INFO:teuthology.orchestra.run.trial196.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-20T21:02:26.469 INFO:teuthology.orchestra.run.trial196.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-20T21:02:26.469 INFO:teuthology.orchestra.run.trial196.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-20T21:02:26.469 INFO:teuthology.orchestra.run.trial196.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-20T21:02:27.042 INFO:teuthology.orchestra.run.trial196.stdout:Discarding blocks...Done. 2026-02-20T21:02:27.044 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial196.front.sepia.ceph.com -o noatime 2026-02-20T21:02:27.044 DEBUG:teuthology.orchestra.run.trial196:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster1-0 2026-02-20T21:02:27.113 DEBUG:teuthology.orchestra.run.trial196:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-0 2026-02-20T21:02:27.181 DEBUG:teuthology.orchestra.run.trial196:> sudo mkdir -p /var/lib/ceph/osd/cluster1-1 2026-02-20T21:02:27.245 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:02:27.246 INFO:tasks.ceph:role: cluster1.osd.1 2026-02-20T21:02:27.246 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial196.front.sepia.ceph.com 2026-02-20T21:02:27.246 DEBUG:teuthology.orchestra.run.trial196:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-20T21:02:27.312 INFO:teuthology.orchestra.run.trial196.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-20T21:02:27.312 INFO:teuthology.orchestra.run.trial196.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-20T21:02:27.313 INFO:teuthology.orchestra.run.trial196.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-20T21:02:27.313 INFO:teuthology.orchestra.run.trial196.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-20T21:02:27.313 INFO:teuthology.orchestra.run.trial196.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-20T21:02:27.313 INFO:teuthology.orchestra.run.trial196.stdout: = sunit=0 swidth=0 blks 2026-02-20T21:02:27.313 INFO:teuthology.orchestra.run.trial196.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-20T21:02:27.313 INFO:teuthology.orchestra.run.trial196.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-20T21:02:27.313 INFO:teuthology.orchestra.run.trial196.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-20T21:02:27.313 INFO:teuthology.orchestra.run.trial196.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-20T21:02:27.869 INFO:teuthology.orchestra.run.trial196.stdout:Discarding blocks...Done. 2026-02-20T21:02:27.871 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial196.front.sepia.ceph.com -o noatime 2026-02-20T21:02:27.871 DEBUG:teuthology.orchestra.run.trial196:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster1-1 2026-02-20T21:02:27.940 DEBUG:teuthology.orchestra.run.trial196:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-1 2026-02-20T21:02:28.007 DEBUG:teuthology.orchestra.run.trial196:> sudo mkdir -p /var/lib/ceph/osd/cluster1-2 2026-02-20T21:02:28.071 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:02:28.071 INFO:tasks.ceph:role: cluster1.osd.2 2026-02-20T21:02:28.071 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial196.front.sepia.ceph.com 2026-02-20T21:02:28.072 DEBUG:teuthology.orchestra.run.trial196:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-20T21:02:28.138 INFO:teuthology.orchestra.run.trial196.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-20T21:02:28.138 INFO:teuthology.orchestra.run.trial196.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-20T21:02:28.138 INFO:teuthology.orchestra.run.trial196.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-20T21:02:28.138 INFO:teuthology.orchestra.run.trial196.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-20T21:02:28.138 INFO:teuthology.orchestra.run.trial196.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-20T21:02:28.138 INFO:teuthology.orchestra.run.trial196.stdout: = sunit=0 swidth=0 blks 2026-02-20T21:02:28.138 INFO:teuthology.orchestra.run.trial196.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-20T21:02:28.138 INFO:teuthology.orchestra.run.trial196.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-20T21:02:28.139 INFO:teuthology.orchestra.run.trial196.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-20T21:02:28.139 INFO:teuthology.orchestra.run.trial196.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-20T21:02:28.699 INFO:teuthology.orchestra.run.trial196.stdout:Discarding blocks...Done. 2026-02-20T21:02:28.701 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial196.front.sepia.ceph.com -o noatime 2026-02-20T21:02:28.702 DEBUG:teuthology.orchestra.run.trial196:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster1-2 2026-02-20T21:02:28.772 DEBUG:teuthology.orchestra.run.trial196:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-2 2026-02-20T21:02:28.839 DEBUG:teuthology.orchestra.run.trial196:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:02:28.917 INFO:teuthology.orchestra.run.trial196.stderr:2026-02-20T21:02:28.914+0000 7fcc42383900 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-0/keyring: can't open /var/lib/ceph/osd/cluster1-0/keyring: (2) No such file or directory 2026-02-20T21:02:28.917 INFO:teuthology.orchestra.run.trial196.stderr:2026-02-20T21:02:28.914+0000 7fcc42383900 -1 created new key in keyring /var/lib/ceph/osd/cluster1-0/keyring 2026-02-20T21:02:28.918 INFO:teuthology.orchestra.run.trial196.stderr:2026-02-20T21:02:28.914+0000 7fcc42383900 -1 bdev(0x556531239800 /var/lib/ceph/osd/cluster1-0/block) open stat got: (1) Operation not permitted 2026-02-20T21:02:28.918 INFO:teuthology.orchestra.run.trial196.stderr:2026-02-20T21:02:28.914+0000 7fcc42383900 -1 bluestore(/var/lib/ceph/osd/cluster1-0) _read_fsid unparsable uuid 2026-02-20T21:02:29.384 DEBUG:teuthology.orchestra.run.trial196:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-0 2026-02-20T21:02:29.449 DEBUG:teuthology.orchestra.run.trial196:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:02:29.529 INFO:teuthology.orchestra.run.trial196.stderr:2026-02-20T21:02:29.526+0000 7faeebf57900 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-1/keyring: can't open /var/lib/ceph/osd/cluster1-1/keyring: (2) No such file or directory 2026-02-20T21:02:29.529 INFO:teuthology.orchestra.run.trial196.stderr:2026-02-20T21:02:29.526+0000 7faeebf57900 -1 created new key in keyring /var/lib/ceph/osd/cluster1-1/keyring 2026-02-20T21:02:29.529 INFO:teuthology.orchestra.run.trial196.stderr:2026-02-20T21:02:29.526+0000 7faeebf57900 -1 bdev(0x556686867800 /var/lib/ceph/osd/cluster1-1/block) open stat got: (1) Operation not permitted 2026-02-20T21:02:29.529 INFO:teuthology.orchestra.run.trial196.stderr:2026-02-20T21:02:29.526+0000 7faeebf57900 -1 bluestore(/var/lib/ceph/osd/cluster1-1) _read_fsid unparsable uuid 2026-02-20T21:02:29.993 DEBUG:teuthology.orchestra.run.trial196:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-1 2026-02-20T21:02:30.057 DEBUG:teuthology.orchestra.run.trial196:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:02:30.136 INFO:teuthology.orchestra.run.trial196.stderr:2026-02-20T21:02:30.133+0000 7ffa7d95b900 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-2/keyring: can't open /var/lib/ceph/osd/cluster1-2/keyring: (2) No such file or directory 2026-02-20T21:02:30.136 INFO:teuthology.orchestra.run.trial196.stderr:2026-02-20T21:02:30.133+0000 7ffa7d95b900 -1 created new key in keyring /var/lib/ceph/osd/cluster1-2/keyring 2026-02-20T21:02:30.136 INFO:teuthology.orchestra.run.trial196.stderr:2026-02-20T21:02:30.133+0000 7ffa7d95b900 -1 bdev(0x56353c0eb800 /var/lib/ceph/osd/cluster1-2/block) open stat got: (1) Operation not permitted 2026-02-20T21:02:30.136 INFO:teuthology.orchestra.run.trial196.stderr:2026-02-20T21:02:30.133+0000 7ffa7d95b900 -1 bluestore(/var/lib/ceph/osd/cluster1-2) _read_fsid unparsable uuid 2026-02-20T21:02:30.600 DEBUG:teuthology.orchestra.run.trial196:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-2 2026-02-20T21:02:30.667 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-20T21:02:30.667 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T21:02:30.667 DEBUG:teuthology.orchestra.run.trial196:> sudo dd if=/var/lib/ceph/mgr/cluster1-x/keyring of=/dev/stdout 2026-02-20T21:02:30.732 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T21:02:30.732 DEBUG:teuthology.orchestra.run.trial196:> sudo dd if=/var/lib/ceph/osd/cluster1-0/keyring of=/dev/stdout 2026-02-20T21:02:30.797 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T21:02:30.797 DEBUG:teuthology.orchestra.run.trial196:> sudo dd if=/var/lib/ceph/osd/cluster1-1/keyring of=/dev/stdout 2026-02-20T21:02:30.861 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T21:02:30.861 DEBUG:teuthology.orchestra.run.trial196:> sudo dd if=/var/lib/ceph/osd/cluster1-2/keyring of=/dev/stdout 2026-02-20T21:02:30.924 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T21:02:30.925 DEBUG:teuthology.orchestra.run.trial196:> dd if=/etc/ceph/cluster1.client.0.keyring of=/dev/stdout 2026-02-20T21:02:30.980 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:30.980 DEBUG:teuthology.orchestra.run.trial199:> dd if=/etc/ceph/cluster1.client.mirror.keyring of=/dev/stdout 2026-02-20T21:02:30.999 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:30.999 DEBUG:teuthology.orchestra.run.trial199:> dd if=/etc/ceph/cluster1.client.mirror.0.keyring of=/dev/stdout 2026-02-20T21:02:31.055 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:31.055 DEBUG:teuthology.orchestra.run.trial199:> dd if=/etc/ceph/cluster1.client.mirror.1.keyring of=/dev/stdout 2026-02-20T21:02:31.111 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:31.111 DEBUG:teuthology.orchestra.run.trial199:> dd if=/etc/ceph/cluster1.client.mirror.2.keyring of=/dev/stdout 2026-02-20T21:02:31.168 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:31.169 DEBUG:teuthology.orchestra.run.trial199:> dd if=/etc/ceph/cluster1.client.mirror.3.keyring of=/dev/stdout 2026-02-20T21:02:31.224 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:31.225 DEBUG:teuthology.orchestra.run.trial199:> dd if=/etc/ceph/cluster1.client.mirror.4.keyring of=/dev/stdout 2026-02-20T21:02:31.281 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:31.282 DEBUG:teuthology.orchestra.run.trial199:> dd if=/etc/ceph/cluster1.client.mirror.5.keyring of=/dev/stdout 2026-02-20T21:02:31.338 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:31.338 DEBUG:teuthology.orchestra.run.trial199:> dd if=/etc/ceph/cluster1.client.mirror.6.keyring of=/dev/stdout 2026-02-20T21:02:31.394 INFO:tasks.ceph:Adding keys to all mons... 2026-02-20T21:02:31.394 DEBUG:teuthology.orchestra.run.trial196:> sudo tee -a /etc/ceph/cluster1.keyring 2026-02-20T21:02:31.418 INFO:teuthology.orchestra.run.trial196.stdout:[mgr.x] 2026-02-20T21:02:31.418 INFO:teuthology.orchestra.run.trial196.stdout: key = AQDhy5hpn1A0KBAAiNe0o2XxUmZgf1YCIQt87A== 2026-02-20T21:02:31.418 INFO:teuthology.orchestra.run.trial196.stdout:[osd.0] 2026-02-20T21:02:31.418 INFO:teuthology.orchestra.run.trial196.stdout: key = AQDky5hpqtGUNhAAs14oskONRhZe35SONYTYyg== 2026-02-20T21:02:31.419 INFO:teuthology.orchestra.run.trial196.stdout:[osd.1] 2026-02-20T21:02:31.419 INFO:teuthology.orchestra.run.trial196.stdout: key = AQDly5hpgItuHxAAgk/PMffR+apx2f6mIOcuRQ== 2026-02-20T21:02:31.419 INFO:teuthology.orchestra.run.trial196.stdout:[osd.2] 2026-02-20T21:02:31.419 INFO:teuthology.orchestra.run.trial196.stdout: key = AQDmy5hp1fAHCBAAzc9PsnxYn6Bvg/Cs4kPABw== 2026-02-20T21:02:31.419 INFO:teuthology.orchestra.run.trial196.stdout:[client.0] 2026-02-20T21:02:31.419 INFO:teuthology.orchestra.run.trial196.stdout: key = AQDhy5hp6o1nKhAAV0gJK5Cvo1f782Uoa1+HDA== 2026-02-20T21:02:31.419 INFO:teuthology.orchestra.run.trial196.stdout:[client.mirror] 2026-02-20T21:02:31.419 INFO:teuthology.orchestra.run.trial196.stdout: key = AQDhy5hpzZRFLRAAZCep4c18uP5QCY+heo+0PA== 2026-02-20T21:02:31.419 INFO:teuthology.orchestra.run.trial196.stdout:[client.mirror.0] 2026-02-20T21:02:31.419 INFO:teuthology.orchestra.run.trial196.stdout: key = AQDhy5hpGBptMhAAXIxDxKmuIWVPT0qg2J8Wfg== 2026-02-20T21:02:31.419 INFO:teuthology.orchestra.run.trial196.stdout:[client.mirror.1] 2026-02-20T21:02:31.420 INFO:teuthology.orchestra.run.trial196.stdout: key = AQDhy5hpc4yWNxAAaR5N8NvCR6cqM2D7xTaoCg== 2026-02-20T21:02:31.420 INFO:teuthology.orchestra.run.trial196.stdout:[client.mirror.2] 2026-02-20T21:02:31.420 INFO:teuthology.orchestra.run.trial196.stdout: key = AQDiy5hpMwAOARAAXeusCfPWgDz0u9Kc6qvs+Q== 2026-02-20T21:02:31.420 INFO:teuthology.orchestra.run.trial196.stdout:[client.mirror.3] 2026-02-20T21:02:31.420 INFO:teuthology.orchestra.run.trial196.stdout: key = AQDiy5hpG/xNBhAAf8aXhKl8gDTMi+kWbkzfMA== 2026-02-20T21:02:31.420 INFO:teuthology.orchestra.run.trial196.stdout:[client.mirror.4] 2026-02-20T21:02:31.420 INFO:teuthology.orchestra.run.trial196.stdout: key = AQDiy5hpMOlwCxAA2svqHAJIU4dXKiKu9Uqxpg== 2026-02-20T21:02:31.420 INFO:teuthology.orchestra.run.trial196.stdout:[client.mirror.5] 2026-02-20T21:02:31.420 INFO:teuthology.orchestra.run.trial196.stdout: key = AQDiy5hpxq6WEBAAe9nMMfjqDxsJRFou19tZsA== 2026-02-20T21:02:31.420 INFO:teuthology.orchestra.run.trial196.stdout:[client.mirror.6] 2026-02-20T21:02:31.420 INFO:teuthology.orchestra.run.trial196.stdout: key = AQDiy5hpYjC8FRAAMjXbeu0X7XY0mBnAFiuphA== 2026-02-20T21:02:31.422 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-20T21:02:31.503 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-20T21:02:31.583 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-20T21:02:31.661 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-20T21:02:31.741 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:02:31.823 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:02:31.907 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:02:31.988 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:02:32.070 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:02:32.152 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:02:32.233 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:02:32.316 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:02:32.398 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:02:32.480 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-20T21:02:32.480 DEBUG:teuthology.orchestra.run.trial196:> sudo mkdir -p /var/lib/ceph/mon/cluster1-a 2026-02-20T21:02:32.544 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster1 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster1.monmap --keyring /etc/ceph/cluster1.keyring 2026-02-20T21:02:32.633 DEBUG:teuthology.orchestra.run.trial196:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster1-a 2026-02-20T21:02:32.658 DEBUG:teuthology.orchestra.run.trial196:> rm -- /home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:02:32.714 INFO:tasks.ceph:Starting mon daemons in cluster cluster1... 2026-02-20T21:02:32.714 INFO:tasks.ceph.cluster1.mon.a:Restarting daemon 2026-02-20T21:02:32.714 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster1 -i a 2026-02-20T21:02:32.756 INFO:tasks.ceph.cluster1.mon.a:Started 2026-02-20T21:02:32.756 INFO:tasks.ceph:Starting mgr daemons in cluster cluster1... 2026-02-20T21:02:32.757 INFO:tasks.ceph.cluster1.mgr.x:Restarting daemon 2026-02-20T21:02:32.757 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster1 -i x 2026-02-20T21:02:32.759 INFO:tasks.ceph.cluster1.mgr.x:Started 2026-02-20T21:02:32.759 DEBUG:tasks.ceph:set 0 configs 2026-02-20T21:02:32.759 DEBUG:teuthology.orchestra.run.trial196:> sudo ceph --cluster cluster1 config dump 2026-02-20T21:02:32.861 INFO:teuthology.orchestra.run.trial196.stdout:WHO MASK LEVEL OPTION VALUE RO 2026-02-20T21:02:32.869 INFO:tasks.ceph:Setting crush tunables to default 2026-02-20T21:02:32.870 DEBUG:teuthology.orchestra.run.trial196:> sudo ceph --cluster cluster1 osd crush tunables default 2026-02-20T21:02:32.973 INFO:teuthology.orchestra.run.trial196.stderr:adjusted tunables profile to default 2026-02-20T21:02:32.981 INFO:tasks.ceph:check_enable_crimson: False 2026-02-20T21:02:32.981 INFO:tasks.ceph:Starting osd daemons in cluster cluster1... 2026-02-20T21:02:32.981 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T21:02:32.981 DEBUG:teuthology.orchestra.run.trial196:> sudo dd if=/var/lib/ceph/osd/cluster1-0/fsid of=/dev/stdout 2026-02-20T21:02:33.006 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T21:02:33.007 DEBUG:teuthology.orchestra.run.trial196:> sudo dd if=/var/lib/ceph/osd/cluster1-1/fsid of=/dev/stdout 2026-02-20T21:02:33.070 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T21:02:33.070 DEBUG:teuthology.orchestra.run.trial196:> sudo dd if=/var/lib/ceph/osd/cluster1-2/fsid of=/dev/stdout 2026-02-20T21:02:33.135 DEBUG:teuthology.orchestra.run.trial196:> sudo ceph --cluster cluster1 osd new 3daca0a3-5551-443d-aa67-7bfefab90bd6 0 2026-02-20T21:02:33.282 INFO:teuthology.orchestra.run.trial196.stdout:0 2026-02-20T21:02:33.291 DEBUG:teuthology.orchestra.run.trial196:> sudo ceph --cluster cluster1 osd new 68aa9c95-119c-4129-83d1-d79efba07215 1 2026-02-20T21:02:33.396 INFO:teuthology.orchestra.run.trial196.stdout:1 2026-02-20T21:02:33.404 DEBUG:teuthology.orchestra.run.trial196:> sudo ceph --cluster cluster1 osd new c09e5039-21ad-44d7-a5c8-b008a89ac5b4 2 2026-02-20T21:02:33.509 INFO:teuthology.orchestra.run.trial196.stdout:2 2026-02-20T21:02:33.518 INFO:tasks.ceph.cluster1.osd.0:Restarting daemon 2026-02-20T21:02:33.518 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 0 2026-02-20T21:02:33.520 INFO:tasks.ceph.cluster1.osd.0:Started 2026-02-20T21:02:33.521 INFO:tasks.ceph.cluster1.osd.1:Restarting daemon 2026-02-20T21:02:33.521 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 1 2026-02-20T21:02:33.523 INFO:tasks.ceph.cluster1.osd.1:Started 2026-02-20T21:02:33.523 INFO:tasks.ceph.cluster1.osd.2:Restarting daemon 2026-02-20T21:02:33.523 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 2 2026-02-20T21:02:33.525 INFO:tasks.ceph.cluster1.osd.2:Started 2026-02-20T21:02:33.526 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-20T21:02:33.796 INFO:teuthology.orchestra.run.trial196.stderr:2026-02-20T21:02:33.794+0000 7ff9d2ffd640 0 --2- 10.20.193.196:0/2217062603 >> [v2:10.20.193.196:3300/0,v1:10.20.193.196:6789/0] conn(0x7ff9d409bc10 0x7ff9d41550a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2026-02-20T21:02:33.797 INFO:tasks.ceph.cluster1.osd.2.trial196.stderr:2026-02-20T21:02:33.794+0000 7f728f4d2900 -1 Falling back to public interface 2026-02-20T21:02:33.811 INFO:tasks.ceph.cluster1.osd.0.trial196.stderr:2026-02-20T21:02:33.809+0000 7fbb4da0d900 -1 Falling back to public interface 2026-02-20T21:02:33.812 INFO:tasks.ceph.cluster1.osd.1.trial196.stderr:2026-02-20T21:02:33.809+0000 7fe46b00d900 -1 Falling back to public interface 2026-02-20T21:02:33.812 INFO:tasks.ceph.cluster1.osd.1.trial196.stderr:2026-02-20T21:02:33.809+0000 7fe4697d0640 -1 PosixStack listen unable to listen on v2:0.0.0.0:6808/0: (98) Address already in use 2026-02-20T21:02:33.828 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:33.829 INFO:teuthology.orchestra.run.trial196.stdout:{"epoch":5,"fsid":"d723a433-cea3-4643-ada0-27ddcd59cc6f","created":"2026-02-20T21:02:32.822606+0000","modified":"2026-02-20T21:02:33.507752+0000","last_up_change":"0.000000","last_in_change":"2026-02-20T21:02:33.507752+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"3daca0a3-5551-443d-aa67-7bfefab90bd6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"68aa9c95-119c-4129-83d1-d79efba07215","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"c09e5039-21ad-44d7-a5c8-b008a89ac5b4","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:02:33.837 INFO:tasks.ceph.ceph_manager.cluster1:[] 2026-02-20T21:02:33.837 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-20T21:02:33.851 INFO:tasks.ceph.cluster1.mgr.x.trial196.stderr:/usr/lib64/python3.9/site-packages/scipy/__init__.py:73: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-20T21:02:33.851 INFO:tasks.ceph.cluster1.mgr.x.trial196.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-20T21:02:33.851 INFO:tasks.ceph.cluster1.mgr.x.trial196.stderr: from numpy import show_config as show_numpy_config 2026-02-20T21:02:34.140 DEBUG:teuthology.orchestra.run.trial196:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-20T21:02:34.234 INFO:teuthology.misc.health.trial196.stdout: 2026-02-20T21:02:34.235 INFO:teuthology.misc.health.trial196.stdout:{"epoch":5,"fsid":"d723a433-cea3-4643-ada0-27ddcd59cc6f","created":"2026-02-20T21:02:32.822606+0000","modified":"2026-02-20T21:02:33.507752+0000","last_up_change":"0.000000","last_in_change":"2026-02-20T21:02:33.507752+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"3daca0a3-5551-443d-aa67-7bfefab90bd6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"68aa9c95-119c-4129-83d1-d79efba07215","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"c09e5039-21ad-44d7-a5c8-b008a89ac5b4","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:02:34.242 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-20T21:02:34.432 INFO:tasks.ceph.cluster1.osd.2.trial196.stderr:2026-02-20T21:02:34.429+0000 7f728f4d2900 -1 osd.2 0 log_to_monitors true 2026-02-20T21:02:34.485 INFO:tasks.ceph.cluster1.osd.0.trial196.stderr:2026-02-20T21:02:34.482+0000 7fbb4da0d900 -1 osd.0 0 log_to_monitors true 2026-02-20T21:02:34.492 INFO:tasks.ceph.cluster1.osd.1.trial196.stderr:2026-02-20T21:02:34.489+0000 7fe46b00d900 -1 osd.1 0 log_to_monitors true 2026-02-20T21:02:35.832 INFO:tasks.ceph.cluster1.osd.1.trial196.stderr:2026-02-20T21:02:35.829+0000 7fe466fcb640 -1 osd.1 0 waiting for initial osdmap 2026-02-20T21:02:35.832 INFO:tasks.ceph.cluster1.osd.0.trial196.stderr:2026-02-20T21:02:35.829+0000 7fbb49174640 -1 osd.0 0 waiting for initial osdmap 2026-02-20T21:02:35.833 INFO:tasks.ceph.cluster1.osd.2.trial196.stderr:2026-02-20T21:02:35.829+0000 7f728b460640 -1 osd.2 0 waiting for initial osdmap 2026-02-20T21:02:35.846 INFO:tasks.ceph.cluster1.osd.0.trial196.stderr:2026-02-20T21:02:35.844+0000 7fbb40772640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-20T21:02:35.847 INFO:tasks.ceph.cluster1.osd.1.trial196.stderr:2026-02-20T21:02:35.844+0000 7fe45ddb7640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-20T21:02:35.847 INFO:tasks.ceph.cluster1.osd.2.trial196.stderr:2026-02-20T21:02:35.844+0000 7f7282a5e640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-20T21:02:36.063 INFO:tasks.ceph.cluster1.mgr.x.trial196.stderr:2026-02-20T21:02:36.060+0000 7fbf6becf640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-20T21:02:40.546 DEBUG:teuthology.orchestra.run.trial196:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-20T21:02:40.720 INFO:teuthology.misc.health.trial196.stdout: 2026-02-20T21:02:40.720 INFO:teuthology.misc.health.trial196.stdout:{"epoch":11,"fsid":"d723a433-cea3-4643-ada0-27ddcd59cc6f","created":"2026-02-20T21:02:32.822606+0000","modified":"2026-02-20T21:02:39.835349+0000","last_up_change":"2026-02-20T21:02:36.830199+0000","last_in_change":"2026-02-20T21:02:33.507752+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:02:37.069741+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"3daca0a3-5551-443d-aa67-7bfefab90bd6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6808","nonce":1534278585},{"type":"v1","addr":"10.20.193.196:6810","nonce":1534278585}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6812","nonce":1534278585},{"type":"v1","addr":"10.20.193.196:6815","nonce":1534278585}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6821","nonce":1534278585},{"type":"v1","addr":"10.20.193.196:6823","nonce":1534278585}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6817","nonce":1534278585},{"type":"v1","addr":"10.20.193.196:6819","nonce":1534278585}]},"public_addr":"10.20.193.196:6810/1534278585","cluster_addr":"10.20.193.196:6815/1534278585","heartbeat_back_addr":"10.20.193.196:6823/1534278585","heartbeat_front_addr":"10.20.193.196:6819/1534278585","state":["exists","up"]},{"osd":1,"uuid":"68aa9c95-119c-4129-83d1-d79efba07215","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6809","nonce":3174935431},{"type":"v1","addr":"10.20.193.196:6811","nonce":3174935431}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6813","nonce":3174935431},{"type":"v1","addr":"10.20.193.196:6814","nonce":3174935431}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6820","nonce":3174935431},{"type":"v1","addr":"10.20.193.196:6822","nonce":3174935431}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6816","nonce":3174935431},{"type":"v1","addr":"10.20.193.196:6818","nonce":3174935431}]},"public_addr":"10.20.193.196:6811/3174935431","cluster_addr":"10.20.193.196:6814/3174935431","heartbeat_back_addr":"10.20.193.196:6822/3174935431","heartbeat_front_addr":"10.20.193.196:6818/3174935431","state":["exists","up"]},{"osd":2,"uuid":"c09e5039-21ad-44d7-a5c8-b008a89ac5b4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6800","nonce":2134765115},{"type":"v1","addr":"10.20.193.196:6801","nonce":2134765115}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6802","nonce":2134765115},{"type":"v1","addr":"10.20.193.196:6803","nonce":2134765115}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6806","nonce":2134765115},{"type":"v1","addr":"10.20.193.196:6807","nonce":2134765115}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6804","nonce":2134765115},{"type":"v1","addr":"10.20.193.196:6805","nonce":2134765115}]},"public_addr":"10.20.193.196:6801/2134765115","cluster_addr":"10.20.193.196:6803/2134765115","heartbeat_back_addr":"10.20.193.196:6807/2134765115","heartbeat_front_addr":"10.20.193.196:6805/2134765115","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:02:35.468565+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:02:35.469621+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:02:35.434924+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:02:40.728 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-20T21:02:40.728 INFO:tasks.ceph:Creating RBD pool 2026-02-20T21:02:40.728 DEBUG:teuthology.orchestra.run.trial196:> sudo ceph --cluster cluster1 osd pool create rbd 8 2026-02-20T21:02:41.842 INFO:teuthology.orchestra.run.trial196.stderr:pool 'rbd' created 2026-02-20T21:02:41.855 DEBUG:teuthology.orchestra.run.trial196:> rbd --cluster cluster1 pool init rbd 2026-02-20T21:02:44.855 INFO:tasks.ceph:Starting mds daemons in cluster cluster1... 2026-02-20T21:02:44.856 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 config log 1 --format=json 2026-02-20T21:02:44.856 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-20T21:02:45.038 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:45.047 INFO:teuthology.orchestra.run.trial196.stdout:[{"version":1,"timestamp":"0.000000","name":"","changes":[]}] 2026-02-20T21:02:45.047 INFO:tasks.ceph_manager:config epoch is 1 2026-02-20T21:02:45.047 INFO:tasks.ceph:Waiting until cluster1 daemons up and pgs clean... 2026-02-20T21:02:45.047 INFO:tasks.ceph.ceph_manager.cluster1:waiting for mgr available 2026-02-20T21:02:45.048 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 mgr dump --format=json 2026-02-20T21:02:45.264 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:45.272 INFO:teuthology.orchestra.run.trial196.stdout:{"epoch":5,"flags":0,"active_gid":4104,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6824","nonce":3542588197},{"type":"v1","addr":"10.20.193.196:6825","nonce":3542588197}]},"active_addr":"10.20.193.196:6825/3542588197","active_change":"2026-02-20T21:02:35.055932+0000","active_mgr_features":4544132024016699391,"available":true,"standbys":[],"modules":["iostat","nfs"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"certificate_automated_rotation_enabled":{"name":"certificate_automated_rotation_enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"This flag controls whether cephadm automatically rotates certificates upon expiration.","long_desc":"","tags":[],"see_also":[]},"certificate_check_debug_mode":{"name":"certificate_check_debug_mode","type":"bool","level":"dev","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"FOR TESTING ONLY: This flag forces the certificate check instead of waiting for certificate_check_period.","long_desc":"","tags":[],"see_also":[]},"certificate_check_period":{"name":"certificate_check_period","type":"int","level":"advanced","flags":0,"default_value":"1","min":"0","max":"30","enum_allowed":[],"desc":"Specifies how often (in days) the certificate should be checked for validity.","long_desc":"","tags":[],"see_also":[]},"certificate_duration_days":{"name":"certificate_duration_days","type":"int","level":"advanced","flags":0,"default_value":"1095","min":"90","max":"3650","enum_allowed":[],"desc":"Specifies the duration of self certificates generated and signed by cephadm root CA","long_desc":"","tags":[],"see_also":[]},"certificate_renewal_threshold_days":{"name":"certificate_renewal_threshold_days","type":"int","level":"advanced","flags":0,"default_value":"30","min":"10","max":"90","enum_allowed":[],"desc":"Specifies the lead time in days to initiate certificate renewal before expiration.","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.28.1","min":"","max":"","enum_allowed":[],"desc":"Alertmanager container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"Elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:12.3.1","min":"","max":"","enum_allowed":[],"desc":"Grafana container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"Haproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.9.1","min":"","max":"","enum_allowed":[],"desc":"Node exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.5","min":"","max":"","enum_allowed":[],"desc":"Nvmeof container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"Oauth2 proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v3.6.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba metrics container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"Snmp gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"stray_daemon_check_interval":{"name":"stray_daemon_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"how frequently cephadm should check for the presence of stray daemons","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROM_ALERT_CREDENTIAL_CACHE_TTL":{"name":"PROM_ALERT_CREDENTIAL_CACHE_TTL","type":"int","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_HOSTNAME_PER_DAEMON":{"name":"RGW_HOSTNAME_PER_DAEMON","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crypto_caller":{"name":"crypto_caller","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"prometheus_tls_secret_name":{"name":"prometheus_tls_secret_name","type":"str","level":"advanced","flags":0,"default_value":"rook-ceph-prometheus-server-tls","min":"","max":"","enum_allowed":[],"desc":"name of tls secret in k8s for prometheus","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_cloning":{"name":"pause_cloning","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_purging":{"name":"pause_purging","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous subvolume purge threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"tentacle":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.196:0","nonce":4119160837}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.196:0","nonce":2651669634}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.196:0","nonce":2016123869}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.196:0","nonce":177980028}]}]} 2026-02-20T21:02:45.276 INFO:tasks.ceph.ceph_manager.cluster1:mgr available! 2026-02-20T21:02:45.276 INFO:tasks.ceph.ceph_manager.cluster1:waiting for all up 2026-02-20T21:02:45.276 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-20T21:02:45.460 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:45.460 INFO:teuthology.orchestra.run.trial196.stdout:{"epoch":15,"fsid":"d723a433-cea3-4643-ada0-27ddcd59cc6f","created":"2026-02-20T21:02:32.822606+0000","modified":"2026-02-20T21:02:44.845203+0000","last_up_change":"2026-02-20T21:02:36.830199+0000","last_in_change":"2026-02-20T21:02:33.507752+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:02:37.069741+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-20T21:02:40.909401+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"3daca0a3-5551-443d-aa67-7bfefab90bd6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6808","nonce":1534278585},{"type":"v1","addr":"10.20.193.196:6810","nonce":1534278585}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6812","nonce":1534278585},{"type":"v1","addr":"10.20.193.196:6815","nonce":1534278585}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6821","nonce":1534278585},{"type":"v1","addr":"10.20.193.196:6823","nonce":1534278585}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6817","nonce":1534278585},{"type":"v1","addr":"10.20.193.196:6819","nonce":1534278585}]},"public_addr":"10.20.193.196:6810/1534278585","cluster_addr":"10.20.193.196:6815/1534278585","heartbeat_back_addr":"10.20.193.196:6823/1534278585","heartbeat_front_addr":"10.20.193.196:6819/1534278585","state":["exists","up"]},{"osd":1,"uuid":"68aa9c95-119c-4129-83d1-d79efba07215","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6809","nonce":3174935431},{"type":"v1","addr":"10.20.193.196:6811","nonce":3174935431}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6813","nonce":3174935431},{"type":"v1","addr":"10.20.193.196:6814","nonce":3174935431}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6820","nonce":3174935431},{"type":"v1","addr":"10.20.193.196:6822","nonce":3174935431}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6816","nonce":3174935431},{"type":"v1","addr":"10.20.193.196:6818","nonce":3174935431}]},"public_addr":"10.20.193.196:6811/3174935431","cluster_addr":"10.20.193.196:6814/3174935431","heartbeat_back_addr":"10.20.193.196:6822/3174935431","heartbeat_front_addr":"10.20.193.196:6818/3174935431","state":["exists","up"]},{"osd":2,"uuid":"c09e5039-21ad-44d7-a5c8-b008a89ac5b4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6800","nonce":2134765115},{"type":"v1","addr":"10.20.193.196:6801","nonce":2134765115}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6802","nonce":2134765115},{"type":"v1","addr":"10.20.193.196:6803","nonce":2134765115}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6806","nonce":2134765115},{"type":"v1","addr":"10.20.193.196:6807","nonce":2134765115}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6804","nonce":2134765115},{"type":"v1","addr":"10.20.193.196:6805","nonce":2134765115}]},"public_addr":"10.20.193.196:6801/2134765115","cluster_addr":"10.20.193.196:6803/2134765115","heartbeat_back_addr":"10.20.193.196:6807/2134765115","heartbeat_front_addr":"10.20.193.196:6805/2134765115","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:02:35.468565+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:02:35.469621+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:02:35.434924+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:02:45.470 INFO:tasks.ceph.ceph_manager.cluster1:all up! 2026-02-20T21:02:45.471 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-20T21:02:45.653 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:45.653 INFO:teuthology.orchestra.run.trial196.stdout:{"epoch":15,"fsid":"d723a433-cea3-4643-ada0-27ddcd59cc6f","created":"2026-02-20T21:02:32.822606+0000","modified":"2026-02-20T21:02:44.845203+0000","last_up_change":"2026-02-20T21:02:36.830199+0000","last_in_change":"2026-02-20T21:02:33.507752+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:02:37.069741+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-20T21:02:40.909401+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"3daca0a3-5551-443d-aa67-7bfefab90bd6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6808","nonce":1534278585},{"type":"v1","addr":"10.20.193.196:6810","nonce":1534278585}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6812","nonce":1534278585},{"type":"v1","addr":"10.20.193.196:6815","nonce":1534278585}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6821","nonce":1534278585},{"type":"v1","addr":"10.20.193.196:6823","nonce":1534278585}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6817","nonce":1534278585},{"type":"v1","addr":"10.20.193.196:6819","nonce":1534278585}]},"public_addr":"10.20.193.196:6810/1534278585","cluster_addr":"10.20.193.196:6815/1534278585","heartbeat_back_addr":"10.20.193.196:6823/1534278585","heartbeat_front_addr":"10.20.193.196:6819/1534278585","state":["exists","up"]},{"osd":1,"uuid":"68aa9c95-119c-4129-83d1-d79efba07215","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6809","nonce":3174935431},{"type":"v1","addr":"10.20.193.196:6811","nonce":3174935431}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6813","nonce":3174935431},{"type":"v1","addr":"10.20.193.196:6814","nonce":3174935431}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6820","nonce":3174935431},{"type":"v1","addr":"10.20.193.196:6822","nonce":3174935431}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6816","nonce":3174935431},{"type":"v1","addr":"10.20.193.196:6818","nonce":3174935431}]},"public_addr":"10.20.193.196:6811/3174935431","cluster_addr":"10.20.193.196:6814/3174935431","heartbeat_back_addr":"10.20.193.196:6822/3174935431","heartbeat_front_addr":"10.20.193.196:6818/3174935431","state":["exists","up"]},{"osd":2,"uuid":"c09e5039-21ad-44d7-a5c8-b008a89ac5b4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6800","nonce":2134765115},{"type":"v1","addr":"10.20.193.196:6801","nonce":2134765115}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6802","nonce":2134765115},{"type":"v1","addr":"10.20.193.196:6803","nonce":2134765115}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6806","nonce":2134765115},{"type":"v1","addr":"10.20.193.196:6807","nonce":2134765115}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6804","nonce":2134765115},{"type":"v1","addr":"10.20.193.196:6805","nonce":2134765115}]},"public_addr":"10.20.193.196:6801/2134765115","cluster_addr":"10.20.193.196:6803/2134765115","heartbeat_back_addr":"10.20.193.196:6807/2134765115","heartbeat_front_addr":"10.20.193.196:6805/2134765115","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:02:35.468565+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:02:35.469621+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:02:35.434924+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:02:45.663 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 flush_pg_stats 2026-02-20T21:02:45.663 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 flush_pg_stats 2026-02-20T21:02:45.664 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 flush_pg_stats 2026-02-20T21:02:45.753 INFO:teuthology.orchestra.run.trial196.stdout:34359738371 2026-02-20T21:02:45.754 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-20T21:02:45.754 INFO:teuthology.orchestra.run.trial196.stdout:34359738371 2026-02-20T21:02:45.755 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-20T21:02:45.756 INFO:teuthology.orchestra.run.trial196.stdout:34359738372 2026-02-20T21:02:45.756 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-20T21:02:45.983 INFO:teuthology.orchestra.run.trial196.stdout:34359738370 2026-02-20T21:02:45.984 INFO:teuthology.orchestra.run.trial196.stdout:34359738371 2026-02-20T21:02:45.989 INFO:teuthology.orchestra.run.trial196.stdout:34359738370 2026-02-20T21:02:45.994 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738370 for osd.1 2026-02-20T21:02:45.995 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738371 for osd.2 2026-02-20T21:02:46.000 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738370 for osd.0 2026-02-20T21:02:46.995 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-20T21:02:46.996 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-20T21:02:47.001 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-20T21:02:47.182 INFO:teuthology.orchestra.run.trial196.stdout:34359738370 2026-02-20T21:02:47.185 INFO:teuthology.orchestra.run.trial196.stdout:34359738371 2026-02-20T21:02:47.193 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738370 for osd.1 2026-02-20T21:02:47.196 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738371 for osd.2 2026-02-20T21:02:47.222 INFO:teuthology.orchestra.run.trial196.stdout:34359738370 2026-02-20T21:02:47.232 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738370 for osd.0 2026-02-20T21:02:48.194 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-20T21:02:48.197 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-20T21:02:48.233 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-20T21:02:48.386 INFO:teuthology.orchestra.run.trial196.stdout:34359738371 2026-02-20T21:02:48.386 INFO:teuthology.orchestra.run.trial196.stdout:34359738372 2026-02-20T21:02:48.396 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738371 for osd.1 2026-02-20T21:02:48.396 DEBUG:teuthology.parallel:result is None 2026-02-20T21:02:48.396 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738372 for osd.2 2026-02-20T21:02:48.397 DEBUG:teuthology.parallel:result is None 2026-02-20T21:02:48.422 INFO:teuthology.orchestra.run.trial196.stdout:34359738371 2026-02-20T21:02:48.432 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738371 for osd.0 2026-02-20T21:02:48.433 DEBUG:teuthology.parallel:result is None 2026-02-20T21:02:48.433 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-20T21:02:48.433 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-20T21:02:48.652 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:48.653 INFO:teuthology.orchestra.run.trial196.stderr:dumped all 2026-02-20T21:02:48.660 INFO:teuthology.orchestra.run.trial196.stdout:{"pg_ready":true,"pg_map":{"version":17,"stamp":"2026-02-20T21:02:47.060380+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":15,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81392,"kb_used_data":512,"kb_used_omap":24,"kb_used_meta":80807,"kb_avail":283034128,"statfs":{"total":289910292480,"available":289826947072,"internally_reserved":0,"allocated":524288,"data_stored":1292257,"data_compressed":67146,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":25025,"internal_metadata":82746943},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.216366"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:02:44.848123+0000","last_change":"2026-02-20T21:02:44.848549+0000","last_active":"2026-02-20T21:02:44.848123+0000","last_peered":"2026-02-20T21:02:44.848123+0000","last_clean":"2026-02-20T21:02:44.848123+0000","last_became_active":"2026-02-20T21:02:42.847423+0000","last_became_peered":"2026-02-20T21:02:42.847423+0000","last_unstale":"2026-02-20T21:02:44.848123+0000","last_undegraded":"2026-02-20T21:02:44.848123+0000","last_fullsized":"2026-02-20T21:02:44.848123+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:11:14.799466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000806229,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:02:44.848527+0000","last_change":"2026-02-20T21:02:44.848660+0000","last_active":"2026-02-20T21:02:44.848527+0000","last_peered":"2026-02-20T21:02:44.848527+0000","last_clean":"2026-02-20T21:02:44.848527+0000","last_became_active":"2026-02-20T21:02:42.847356+0000","last_became_peered":"2026-02-20T21:02:42.847356+0000","last_unstale":"2026-02-20T21:02:44.848527+0000","last_undegraded":"2026-02-20T21:02:44.848527+0000","last_fullsized":"2026-02-20T21:02:44.848527+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:12:14.023391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00098168499999999994,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:02:44.848314+0000","last_change":"2026-02-20T21:02:44.848481+0000","last_active":"2026-02-20T21:02:44.848314+0000","last_peered":"2026-02-20T21:02:44.848314+0000","last_clean":"2026-02-20T21:02:44.848314+0000","last_became_active":"2026-02-20T21:02:42.847430+0000","last_became_peered":"2026-02-20T21:02:42.847430+0000","last_unstale":"2026-02-20T21:02:44.848314+0000","last_undegraded":"2026-02-20T21:02:44.848314+0000","last_fullsized":"2026-02-20T21:02:44.848314+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:35:32.634569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081569699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:02:44.847882+0000","last_change":"2026-02-20T21:02:44.848066+0000","last_active":"2026-02-20T21:02:44.847882+0000","last_peered":"2026-02-20T21:02:44.847882+0000","last_clean":"2026-02-20T21:02:44.847882+0000","last_became_active":"2026-02-20T21:02:42.847480+0000","last_became_peered":"2026-02-20T21:02:42.847480+0000","last_unstale":"2026-02-20T21:02:44.847882+0000","last_undegraded":"2026-02-20T21:02:44.847882+0000","last_fullsized":"2026-02-20T21:02:44.847882+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:56:33.514445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045216000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"15'2","reported_seq":22,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:02:44.849371+0000","last_change":"2026-02-20T21:02:44.849371+0000","last_active":"2026-02-20T21:02:44.849371+0000","last_peered":"2026-02-20T21:02:44.849371+0000","last_clean":"2026-02-20T21:02:44.849371+0000","last_became_active":"2026-02-20T21:02:42.847714+0000","last_became_peered":"2026-02-20T21:02:42.847714+0000","last_unstale":"2026-02-20T21:02:44.849371+0000","last_undegraded":"2026-02-20T21:02:44.849371+0000","last_fullsized":"2026-02-20T21:02:44.849371+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:47:39.564991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089537500000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:02:45.359865+0000","last_change":"2026-02-20T21:02:45.359962+0000","last_active":"2026-02-20T21:02:45.359865+0000","last_peered":"2026-02-20T21:02:45.359865+0000","last_clean":"2026-02-20T21:02:45.359865+0000","last_became_active":"2026-02-20T21:02:42.847711+0000","last_became_peered":"2026-02-20T21:02:42.847711+0000","last_unstale":"2026-02-20T21:02:45.359865+0000","last_undegraded":"2026-02-20T21:02:45.359865+0000","last_fullsized":"2026-02-20T21:02:45.359865+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:00:52.446182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023752199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:02:45.359901+0000","last_change":"2026-02-20T21:02:45.360049+0000","last_active":"2026-02-20T21:02:45.359901+0000","last_peered":"2026-02-20T21:02:45.359901+0000","last_clean":"2026-02-20T21:02:45.359901+0000","last_became_active":"2026-02-20T21:02:42.847731+0000","last_became_peered":"2026-02-20T21:02:42.847731+0000","last_unstale":"2026-02-20T21:02:45.359901+0000","last_undegraded":"2026-02-20T21:02:45.359901+0000","last_fullsized":"2026-02-20T21:02:45.359901+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:27:20.095721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000272386,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"13'1","reported_seq":21,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:02:44.847928+0000","last_change":"2026-02-20T21:02:44.848179+0000","last_active":"2026-02-20T21:02:44.847928+0000","last_peered":"2026-02-20T21:02:44.847928+0000","last_clean":"2026-02-20T21:02:44.847928+0000","last_became_active":"2026-02-20T21:02:42.847080+0000","last_became_peered":"2026-02-20T21:02:42.847080+0000","last_unstale":"2026-02-20T21:02:44.847928+0000","last_undegraded":"2026-02-20T21:02:44.847928+0000","last_fullsized":"2026-02-20T21:02:44.847928+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:43:24.107388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057432800000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'72","reported_seq":110,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:02:44.847433+0000","last_change":"2026-02-20T21:02:39.094532+0000","last_active":"2026-02-20T21:02:44.847433+0000","last_peered":"2026-02-20T21:02:44.847433+0000","last_clean":"2026-02-20T21:02:44.847433+0000","last_became_active":"2026-02-20T21:02:39.093868+0000","last_became_peered":"2026-02-20T21:02:39.093868+0000","last_unstale":"2026-02-20T21:02:44.847433+0000","last_undegraded":"2026-02-20T21:02:44.847433+0000","last_fullsized":"2026-02-20T21:02:44.847433+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:37.832019+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:37.832019+0000","last_clean_scrub_stamp":"2026-02-20T21:02:37.832019+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:52:04.367262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67146,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738372,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27080,"kb_used_data":120,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344760,"statfs":{"total":96636764160,"available":96609034240,"internally_reserved":0,"allocated":122880,"data_stored":34785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738371,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27156,"kb_used_data":196,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344684,"statfs":{"total":96636764160,"available":96608956416,"internally_reserved":0,"allocated":200704,"data_stored":628736,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738371,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27156,"kb_used_data":196,"kb_used_omap":8,"kb_used_meta":26935,"kb_avail":94344684,"statfs":{"total":96636764160,"available":96608956416,"internally_reserved":0,"allocated":200704,"data_stored":628736,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":8775,"internal_metadata":27581881},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:02:48.662 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-20T21:02:48.841 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:48.842 INFO:teuthology.orchestra.run.trial196.stderr:dumped all 2026-02-20T21:02:48.850 INFO:teuthology.orchestra.run.trial196.stdout:{"pg_ready":true,"pg_map":{"version":17,"stamp":"2026-02-20T21:02:47.060380+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":15,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81392,"kb_used_data":512,"kb_used_omap":24,"kb_used_meta":80807,"kb_avail":283034128,"statfs":{"total":289910292480,"available":289826947072,"internally_reserved":0,"allocated":524288,"data_stored":1292257,"data_compressed":67146,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":25025,"internal_metadata":82746943},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.216366"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:02:44.848123+0000","last_change":"2026-02-20T21:02:44.848549+0000","last_active":"2026-02-20T21:02:44.848123+0000","last_peered":"2026-02-20T21:02:44.848123+0000","last_clean":"2026-02-20T21:02:44.848123+0000","last_became_active":"2026-02-20T21:02:42.847423+0000","last_became_peered":"2026-02-20T21:02:42.847423+0000","last_unstale":"2026-02-20T21:02:44.848123+0000","last_undegraded":"2026-02-20T21:02:44.848123+0000","last_fullsized":"2026-02-20T21:02:44.848123+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:11:14.799466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000806229,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:02:44.848527+0000","last_change":"2026-02-20T21:02:44.848660+0000","last_active":"2026-02-20T21:02:44.848527+0000","last_peered":"2026-02-20T21:02:44.848527+0000","last_clean":"2026-02-20T21:02:44.848527+0000","last_became_active":"2026-02-20T21:02:42.847356+0000","last_became_peered":"2026-02-20T21:02:42.847356+0000","last_unstale":"2026-02-20T21:02:44.848527+0000","last_undegraded":"2026-02-20T21:02:44.848527+0000","last_fullsized":"2026-02-20T21:02:44.848527+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:12:14.023391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00098168499999999994,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:02:44.848314+0000","last_change":"2026-02-20T21:02:44.848481+0000","last_active":"2026-02-20T21:02:44.848314+0000","last_peered":"2026-02-20T21:02:44.848314+0000","last_clean":"2026-02-20T21:02:44.848314+0000","last_became_active":"2026-02-20T21:02:42.847430+0000","last_became_peered":"2026-02-20T21:02:42.847430+0000","last_unstale":"2026-02-20T21:02:44.848314+0000","last_undegraded":"2026-02-20T21:02:44.848314+0000","last_fullsized":"2026-02-20T21:02:44.848314+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:35:32.634569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081569699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:02:44.847882+0000","last_change":"2026-02-20T21:02:44.848066+0000","last_active":"2026-02-20T21:02:44.847882+0000","last_peered":"2026-02-20T21:02:44.847882+0000","last_clean":"2026-02-20T21:02:44.847882+0000","last_became_active":"2026-02-20T21:02:42.847480+0000","last_became_peered":"2026-02-20T21:02:42.847480+0000","last_unstale":"2026-02-20T21:02:44.847882+0000","last_undegraded":"2026-02-20T21:02:44.847882+0000","last_fullsized":"2026-02-20T21:02:44.847882+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:56:33.514445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045216000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"15'2","reported_seq":22,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:02:44.849371+0000","last_change":"2026-02-20T21:02:44.849371+0000","last_active":"2026-02-20T21:02:44.849371+0000","last_peered":"2026-02-20T21:02:44.849371+0000","last_clean":"2026-02-20T21:02:44.849371+0000","last_became_active":"2026-02-20T21:02:42.847714+0000","last_became_peered":"2026-02-20T21:02:42.847714+0000","last_unstale":"2026-02-20T21:02:44.849371+0000","last_undegraded":"2026-02-20T21:02:44.849371+0000","last_fullsized":"2026-02-20T21:02:44.849371+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:47:39.564991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089537500000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:02:45.359865+0000","last_change":"2026-02-20T21:02:45.359962+0000","last_active":"2026-02-20T21:02:45.359865+0000","last_peered":"2026-02-20T21:02:45.359865+0000","last_clean":"2026-02-20T21:02:45.359865+0000","last_became_active":"2026-02-20T21:02:42.847711+0000","last_became_peered":"2026-02-20T21:02:42.847711+0000","last_unstale":"2026-02-20T21:02:45.359865+0000","last_undegraded":"2026-02-20T21:02:45.359865+0000","last_fullsized":"2026-02-20T21:02:45.359865+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:00:52.446182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023752199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:02:45.359901+0000","last_change":"2026-02-20T21:02:45.360049+0000","last_active":"2026-02-20T21:02:45.359901+0000","last_peered":"2026-02-20T21:02:45.359901+0000","last_clean":"2026-02-20T21:02:45.359901+0000","last_became_active":"2026-02-20T21:02:42.847731+0000","last_became_peered":"2026-02-20T21:02:42.847731+0000","last_unstale":"2026-02-20T21:02:45.359901+0000","last_undegraded":"2026-02-20T21:02:45.359901+0000","last_fullsized":"2026-02-20T21:02:45.359901+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:27:20.095721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000272386,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"13'1","reported_seq":21,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:02:44.847928+0000","last_change":"2026-02-20T21:02:44.848179+0000","last_active":"2026-02-20T21:02:44.847928+0000","last_peered":"2026-02-20T21:02:44.847928+0000","last_clean":"2026-02-20T21:02:44.847928+0000","last_became_active":"2026-02-20T21:02:42.847080+0000","last_became_peered":"2026-02-20T21:02:42.847080+0000","last_unstale":"2026-02-20T21:02:44.847928+0000","last_undegraded":"2026-02-20T21:02:44.847928+0000","last_fullsized":"2026-02-20T21:02:44.847928+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:43:24.107388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057432800000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'72","reported_seq":110,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:02:44.847433+0000","last_change":"2026-02-20T21:02:39.094532+0000","last_active":"2026-02-20T21:02:44.847433+0000","last_peered":"2026-02-20T21:02:44.847433+0000","last_clean":"2026-02-20T21:02:44.847433+0000","last_became_active":"2026-02-20T21:02:39.093868+0000","last_became_peered":"2026-02-20T21:02:39.093868+0000","last_unstale":"2026-02-20T21:02:44.847433+0000","last_undegraded":"2026-02-20T21:02:44.847433+0000","last_fullsized":"2026-02-20T21:02:44.847433+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:37.832019+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:37.832019+0000","last_clean_scrub_stamp":"2026-02-20T21:02:37.832019+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:52:04.367262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67146,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738372,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27080,"kb_used_data":120,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344760,"statfs":{"total":96636764160,"available":96609034240,"internally_reserved":0,"allocated":122880,"data_stored":34785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738371,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27156,"kb_used_data":196,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344684,"statfs":{"total":96636764160,"available":96608956416,"internally_reserved":0,"allocated":200704,"data_stored":628736,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738371,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27156,"kb_used_data":196,"kb_used_omap":8,"kb_used_meta":26935,"kb_avail":94344684,"statfs":{"total":96636764160,"available":96608956416,"internally_reserved":0,"allocated":200704,"data_stored":628736,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":8775,"internal_metadata":27581881},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:02:48.850 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-20T21:02:48.850 INFO:tasks.ceph:Waiting until ceph cluster cluster1 is healthy... 2026-02-20T21:02:48.850 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy 2026-02-20T21:02:48.850 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 health --format=json 2026-02-20T21:02:49.060 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:49.060 INFO:teuthology.orchestra.run.trial196.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-20T21:02:49.068 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy done 2026-02-20T21:02:49.069 INFO:teuthology.run_tasks:Running task ceph... 2026-02-20T21:02:49.080 INFO:tasks.ceph:Creating ceph cluster cluster2... 2026-02-20T21:02:49.080 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'snappy', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-20T21:02:49.080 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61415', 'branch': 'wip-pdonnell-testing-20260219.182737-tentacle', 'description': 'rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-snappy supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-workunit-fast-diff}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '61415', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run the rbd_mirror.sh workunit to test the rbd-mirror daemon in snapshot mode'}], 'name': 'yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial', 'no_nested_subset': False, 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'wip-pdonnell-testing-20260219.182737-tentacle'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'snappy', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-pdonnell-testing-20260219.182737-tentacle', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 7161, 'sha1': 'f7870454149f22574086dd2ff55d050585136531', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': 'wip-pdonnell-testing-20260219.182737-tentacle', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_f7870454149f22574086dd2ff55d050585136531/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'f7870454149f22574086dd2ff55d050585136531', 'targets': {'trial196.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNXSz/oleXj4WCyNHwLu/9imobgWwskZJuDoZU2v2KihsuN8A40ibPZsGroPVka9yfVxYbjvvfMQvHwms8hWr/I=', 'trial199.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPM+ZdBRamPjFyytTKVlvfP6NMc801M2vKxXYLe18YXGDgXMyw4c98dOBWD/E0Bn+Cp7UqFHFWoxeALuafPFuco='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'snappy', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, {'ceph': {'cluster': 'cluster2', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'snappy', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror.sh']}, 'env': {'CEPH_ARGS': '', 'RBD_IMAGE_FEATURES': 'layering,exclusive-lock,object-map,fast-diff', 'RBD_MIRROR_CONFIG_KEY': '1', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1'}}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-20_16:15:01', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.1540699'} 2026-02-20T21:02:49.081 DEBUG:teuthology.orchestra.run.trial196:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-20T21:02:49.084 DEBUG:teuthology.orchestra.run.trial199:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-20T21:02:49.102 DEBUG:teuthology.orchestra.run.trial196:> sudo install -d -m0777 -- /var/run/ceph 2026-02-20T21:02:49.141 DEBUG:teuthology.orchestra.run.trial199:> sudo install -d -m0777 -- /var/run/ceph 2026-02-20T21:02:49.167 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:49.168 DEBUG:teuthology.orchestra.run.trial199:> dd if=/scratch_devs of=/dev/stdout 2026-02-20T21:02:49.225 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-20T21:02:49.225 DEBUG:teuthology.orchestra.run.trial199:> stat /dev/vg_nvme/lv_1 2026-02-20T21:02:49.282 INFO:teuthology.orchestra.run.trial199.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-20T21:02:49.282 INFO:teuthology.orchestra.run.trial199.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:02:49.282 INFO:teuthology.orchestra.run.trial199.stdout:Device: 6h/6d Inode: 907 Links: 1 2026-02-20T21:02:49.282 INFO:teuthology.orchestra.run.trial199.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:02:49.282 INFO:teuthology.orchestra.run.trial199.stdout:Context: system_u:object_r:device_t:s0 2026-02-20T21:02:49.282 INFO:teuthology.orchestra.run.trial199.stdout:Access: 2026-02-20 21:02:07.680577744 +0000 2026-02-20T21:02:49.282 INFO:teuthology.orchestra.run.trial199.stdout:Modify: 2026-02-20 21:00:51.260126631 +0000 2026-02-20T21:02:49.283 INFO:teuthology.orchestra.run.trial199.stdout:Change: 2026-02-20 21:00:51.260126631 +0000 2026-02-20T21:02:49.283 INFO:teuthology.orchestra.run.trial199.stdout: Birth: 2026-02-20 21:00:51.260126631 +0000 2026-02-20T21:02:49.283 DEBUG:teuthology.orchestra.run.trial199:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-20T21:02:49.345 INFO:teuthology.orchestra.run.trial199.stderr:1+0 records in 2026-02-20T21:02:49.346 INFO:teuthology.orchestra.run.trial199.stderr:1+0 records out 2026-02-20T21:02:49.346 INFO:teuthology.orchestra.run.trial199.stderr:512 bytes copied, 0.000164946 s, 3.1 MB/s 2026-02-20T21:02:49.347 DEBUG:teuthology.orchestra.run.trial199:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-20T21:02:49.405 DEBUG:teuthology.orchestra.run.trial199:> stat /dev/vg_nvme/lv_2 2026-02-20T21:02:49.462 INFO:teuthology.orchestra.run.trial199.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-20T21:02:49.462 INFO:teuthology.orchestra.run.trial199.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:02:49.463 INFO:teuthology.orchestra.run.trial199.stdout:Device: 6h/6d Inode: 938 Links: 1 2026-02-20T21:02:49.463 INFO:teuthology.orchestra.run.trial199.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:02:49.463 INFO:teuthology.orchestra.run.trial199.stdout:Context: system_u:object_r:device_t:s0 2026-02-20T21:02:49.463 INFO:teuthology.orchestra.run.trial199.stdout:Access: 2026-02-20 21:02:07.680577744 +0000 2026-02-20T21:02:49.463 INFO:teuthology.orchestra.run.trial199.stdout:Modify: 2026-02-20 21:00:51.499125479 +0000 2026-02-20T21:02:49.463 INFO:teuthology.orchestra.run.trial199.stdout:Change: 2026-02-20 21:00:51.499125479 +0000 2026-02-20T21:02:49.463 INFO:teuthology.orchestra.run.trial199.stdout: Birth: 2026-02-20 21:00:51.499125479 +0000 2026-02-20T21:02:49.463 DEBUG:teuthology.orchestra.run.trial199:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-20T21:02:49.526 INFO:teuthology.orchestra.run.trial199.stderr:1+0 records in 2026-02-20T21:02:49.526 INFO:teuthology.orchestra.run.trial199.stderr:1+0 records out 2026-02-20T21:02:49.526 INFO:teuthology.orchestra.run.trial199.stderr:512 bytes copied, 0.000141232 s, 3.6 MB/s 2026-02-20T21:02:49.528 DEBUG:teuthology.orchestra.run.trial199:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-20T21:02:49.587 DEBUG:teuthology.orchestra.run.trial199:> stat /dev/vg_nvme/lv_3 2026-02-20T21:02:49.645 INFO:teuthology.orchestra.run.trial199.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-20T21:02:49.645 INFO:teuthology.orchestra.run.trial199.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:02:49.645 INFO:teuthology.orchestra.run.trial199.stdout:Device: 6h/6d Inode: 963 Links: 1 2026-02-20T21:02:49.645 INFO:teuthology.orchestra.run.trial199.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:02:49.645 INFO:teuthology.orchestra.run.trial199.stdout:Context: system_u:object_r:device_t:s0 2026-02-20T21:02:49.645 INFO:teuthology.orchestra.run.trial199.stdout:Access: 2026-02-20 21:02:07.680577744 +0000 2026-02-20T21:02:49.646 INFO:teuthology.orchestra.run.trial199.stdout:Modify: 2026-02-20 21:00:51.726124386 +0000 2026-02-20T21:02:49.646 INFO:teuthology.orchestra.run.trial199.stdout:Change: 2026-02-20 21:00:51.726124386 +0000 2026-02-20T21:02:49.646 INFO:teuthology.orchestra.run.trial199.stdout: Birth: 2026-02-20 21:00:51.726124386 +0000 2026-02-20T21:02:49.646 DEBUG:teuthology.orchestra.run.trial199:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-20T21:02:49.707 INFO:teuthology.orchestra.run.trial199.stderr:1+0 records in 2026-02-20T21:02:49.707 INFO:teuthology.orchestra.run.trial199.stderr:1+0 records out 2026-02-20T21:02:49.707 INFO:teuthology.orchestra.run.trial199.stderr:512 bytes copied, 0.000143827 s, 3.6 MB/s 2026-02-20T21:02:49.709 DEBUG:teuthology.orchestra.run.trial199:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-20T21:02:49.767 DEBUG:teuthology.orchestra.run.trial199:> stat /dev/vg_nvme/lv_4 2026-02-20T21:02:49.824 INFO:teuthology.orchestra.run.trial199.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-20T21:02:49.824 INFO:teuthology.orchestra.run.trial199.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:02:49.824 INFO:teuthology.orchestra.run.trial199.stdout:Device: 6h/6d Inode: 991 Links: 1 2026-02-20T21:02:49.824 INFO:teuthology.orchestra.run.trial199.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:02:49.824 INFO:teuthology.orchestra.run.trial199.stdout:Context: system_u:object_r:device_t:s0 2026-02-20T21:02:49.824 INFO:teuthology.orchestra.run.trial199.stdout:Access: 2026-02-20 21:02:07.681577729 +0000 2026-02-20T21:02:49.824 INFO:teuthology.orchestra.run.trial199.stdout:Modify: 2026-02-20 21:00:51.967123226 +0000 2026-02-20T21:02:49.825 INFO:teuthology.orchestra.run.trial199.stdout:Change: 2026-02-20 21:00:51.967123226 +0000 2026-02-20T21:02:49.825 INFO:teuthology.orchestra.run.trial199.stdout: Birth: 2026-02-20 21:00:51.967123226 +0000 2026-02-20T21:02:49.825 DEBUG:teuthology.orchestra.run.trial199:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-20T21:02:49.888 INFO:teuthology.orchestra.run.trial199.stderr:1+0 records in 2026-02-20T21:02:49.888 INFO:teuthology.orchestra.run.trial199.stderr:1+0 records out 2026-02-20T21:02:49.888 INFO:teuthology.orchestra.run.trial199.stderr:512 bytes copied, 0.000143837 s, 3.6 MB/s 2026-02-20T21:02:49.889 DEBUG:teuthology.orchestra.run.trial199:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-20T21:02:49.946 INFO:tasks.ceph:osd dev map: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:02:49.947 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial199.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-20T21:02:49.947 INFO:tasks.ceph:Generating config... 2026-02-20T21:02:49.949 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-20T21:02:49.949 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-20T21:02:49.949 INFO:tasks.ceph:[client] log to stderr = False 2026-02-20T21:02:49.949 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-20T21:02:49.950 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:49.950 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:49.950 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:49.950 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:49.950 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:49.950 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:49.950 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:49.950 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:49.950 INFO:tasks.ceph:[client.mirror.4] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:49.950 INFO:tasks.ceph:[client.mirror.4] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:49.950 INFO:tasks.ceph:[client.mirror.5] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:49.951 INFO:tasks.ceph:[client.mirror.5] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:49.951 INFO:tasks.ceph:[client.mirror.6] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:49.951 INFO:tasks.ceph:[client.mirror.6] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:49.951 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-20T21:02:49.951 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-20T21:02:49.951 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-20T21:02:49.951 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-20T21:02:49.951 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-20T21:02:49.951 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-20T21:02:49.951 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-20T21:02:49.951 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-20T21:02:49.951 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-20T21:02:49.952 INFO:tasks.ceph:[osd] bluestore compression algorithm = snappy 2026-02-20T21:02:49.952 INFO:tasks.ceph:[osd] bluestore compression mode = aggressive 2026-02-20T21:02:49.952 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-20T21:02:49.952 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-20T21:02:49.952 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-20T21:02:49.952 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-20T21:02:49.952 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-20T21:02:49.952 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-20T21:02:49.952 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-20T21:02:49.952 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-20T21:02:49.952 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-20T21:02:49.952 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-20T21:02:49.953 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-20T21:02:49.953 INFO:tasks.ceph:Setting up cluster2.mon.a... 2026-02-20T21:02:49.953 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster2.keyring 2026-02-20T21:02:50.020 INFO:teuthology.orchestra.run.trial199.stdout:creating /etc/ceph/cluster2.keyring 2026-02-20T21:02:50.023 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster2.keyring 2026-02-20T21:02:50.101 DEBUG:teuthology.orchestra.run.trial199:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-20T21:02:50.166 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster2.mon.a', '10.20.193.199')] 2026-02-20T21:02:50.167 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.199', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluestore block size': 96636764160, 'bluestore compression algorithm': 'snappy', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-20T21:02:50.168 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:50.169 DEBUG:teuthology.orchestra.run.trial199:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-20T21:02:50.223 DEBUG:teuthology.orchestra.run.trial199:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.199 --print /home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:02:50.294 INFO:teuthology.orchestra.run.trial199.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:02:50.294 INFO:teuthology.orchestra.run.trial199.stdout:monmaptool: generated fsid f480090c-6109-4036-a7fa-8e7cbf4099eb 2026-02-20T21:02:50.294 INFO:teuthology.orchestra.run.trial199.stdout:setting min_mon_release = tentacle 2026-02-20T21:02:50.295 INFO:teuthology.orchestra.run.trial199.stdout:epoch 0 2026-02-20T21:02:50.295 INFO:teuthology.orchestra.run.trial199.stdout:fsid f480090c-6109-4036-a7fa-8e7cbf4099eb 2026-02-20T21:02:50.295 INFO:teuthology.orchestra.run.trial199.stdout:last_changed 2026-02-20T21:02:50.292551+0000 2026-02-20T21:02:50.295 INFO:teuthology.orchestra.run.trial199.stdout:created 2026-02-20T21:02:50.292551+0000 2026-02-20T21:02:50.295 INFO:teuthology.orchestra.run.trial199.stdout:min_mon_release 20 (tentacle) 2026-02-20T21:02:50.295 INFO:teuthology.orchestra.run.trial199.stdout:election_strategy: 1 2026-02-20T21:02:50.295 INFO:teuthology.orchestra.run.trial199.stdout:0: [v2:10.20.193.199:3300/0,v1:10.20.193.199:6789/0] mon.a 2026-02-20T21:02:50.295 INFO:teuthology.orchestra.run.trial199.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster2.monmap (1 monitors) 2026-02-20T21:02:50.296 DEBUG:teuthology.orchestra.run.trial199:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-20T21:02:50.353 INFO:tasks.ceph:Writing /etc/ceph/cluster2.conf for FSID f480090c-6109-4036-a7fa-8e7cbf4099eb... 2026-02-20T21:02:50.354 DEBUG:teuthology.orchestra.run.trial196:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-20T21:02:50.357 DEBUG:teuthology.orchestra.run.trial199:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-20T21:02:50.396 INFO:teuthology.orchestra.run.trial196.stdout:[global] 2026-02-20T21:02:50.396 INFO:teuthology.orchestra.run.trial196.stdout: chdir = "" 2026-02-20T21:02:50.396 INFO:teuthology.orchestra.run.trial196.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-20T21:02:50.397 INFO:teuthology.orchestra.run.trial196.stdout: auth supported = cephx 2026-02-20T21:02:50.397 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.397 INFO:teuthology.orchestra.run.trial196.stdout: filestore xattr use omap = true 2026-02-20T21:02:50.397 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.397 INFO:teuthology.orchestra.run.trial196.stdout: mon clock drift allowed = 1.000 2026-02-20T21:02:50.397 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.397 INFO:teuthology.orchestra.run.trial196.stdout: osd crush chooseleaf type = 0 2026-02-20T21:02:50.397 INFO:teuthology.orchestra.run.trial196.stdout: auth debug = true 2026-02-20T21:02:50.397 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.397 INFO:teuthology.orchestra.run.trial196.stdout: ms die on old message = true 2026-02-20T21:02:50.397 INFO:teuthology.orchestra.run.trial196.stdout: ms die on bug = true 2026-02-20T21:02:50.397 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.398 INFO:teuthology.orchestra.run.trial196.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-20T21:02:50.398 INFO:teuthology.orchestra.run.trial196.stdout: mon pg warn max object skew = 0 2026-02-20T21:02:50.398 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.398 INFO:teuthology.orchestra.run.trial196.stdout: # disable pg_autoscaler by default for new pools 2026-02-20T21:02:50.398 INFO:teuthology.orchestra.run.trial196.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-20T21:02:50.398 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.398 INFO:teuthology.orchestra.run.trial196.stdout: osd pool default size = 2 2026-02-20T21:02:50.398 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.398 INFO:teuthology.orchestra.run.trial196.stdout: mon osd allow primary affinity = true 2026-02-20T21:02:50.398 INFO:teuthology.orchestra.run.trial196.stdout: mon osd allow pg remap = true 2026-02-20T21:02:50.399 INFO:teuthology.orchestra.run.trial196.stdout: mon warn on legacy crush tunables = false 2026-02-20T21:02:50.399 INFO:teuthology.orchestra.run.trial196.stdout: mon warn on crush straw calc version zero = false 2026-02-20T21:02:50.399 INFO:teuthology.orchestra.run.trial196.stdout: mon warn on no sortbitwise = false 2026-02-20T21:02:50.399 INFO:teuthology.orchestra.run.trial196.stdout: mon warn on osd down out interval zero = false 2026-02-20T21:02:50.399 INFO:teuthology.orchestra.run.trial196.stdout: mon warn on too few osds = false 2026-02-20T21:02:50.399 INFO:teuthology.orchestra.run.trial196.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-20T21:02:50.399 INFO:teuthology.orchestra.run.trial196.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-20T21:02:50.399 INFO:teuthology.orchestra.run.trial196.stdout: mon_allow_pool_size_one = true 2026-02-20T21:02:50.399 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.399 INFO:teuthology.orchestra.run.trial196.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-20T21:02:50.399 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.400 INFO:teuthology.orchestra.run.trial196.stdout: osd default data pool replay window = 5 2026-02-20T21:02:50.400 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.400 INFO:teuthology.orchestra.run.trial196.stdout: mon allow pool delete = true 2026-02-20T21:02:50.400 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.400 INFO:teuthology.orchestra.run.trial196.stdout: mon cluster log file level = debug 2026-02-20T21:02:50.400 INFO:teuthology.orchestra.run.trial196.stdout: debug asserts on shutdown = true 2026-02-20T21:02:50.400 INFO:teuthology.orchestra.run.trial196.stdout: mon health detail to clog = false 2026-02-20T21:02:50.400 INFO:teuthology.orchestra.run.trial196.stdout: mon host = 10.20.193.199 2026-02-20T21:02:50.400 INFO:teuthology.orchestra.run.trial196.stdout: mon client directed command retry = 5 2026-02-20T21:02:50.400 INFO:teuthology.orchestra.run.trial196.stdout: mon warn on pool no app = False 2026-02-20T21:02:50.400 INFO:teuthology.orchestra.run.trial196.stdout: ms inject socket failures = 5000 2026-02-20T21:02:50.400 INFO:teuthology.orchestra.run.trial196.stdout: fsid = f480090c-6109-4036-a7fa-8e7cbf4099eb 2026-02-20T21:02:50.401 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.401 INFO:teuthology.orchestra.run.trial196.stdout:[osd] 2026-02-20T21:02:50.401 INFO:teuthology.orchestra.run.trial196.stdout: osd journal size = 100 2026-02-20T21:02:50.401 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.401 INFO:teuthology.orchestra.run.trial196.stdout: osd scrub load threshold = 5.0 2026-02-20T21:02:50.401 INFO:teuthology.orchestra.run.trial196.stdout: osd scrub max interval = 600 2026-02-20T21:02:50.401 INFO:teuthology.orchestra.run.trial196.stdout: osd mclock profile = high_recovery_ops 2026-02-20T21:02:50.401 INFO:teuthology.orchestra.run.trial196.stdout: osd mclock skip benchmark = true 2026-02-20T21:02:50.401 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.401 INFO:teuthology.orchestra.run.trial196.stdout: osd recover clone overlap = true 2026-02-20T21:02:50.401 INFO:teuthology.orchestra.run.trial196.stdout: osd recovery max chunk = 1048576 2026-02-20T21:02:50.402 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.402 INFO:teuthology.orchestra.run.trial196.stdout: osd debug shutdown = true 2026-02-20T21:02:50.402 INFO:teuthology.orchestra.run.trial196.stdout: osd debug op order = true 2026-02-20T21:02:50.402 INFO:teuthology.orchestra.run.trial196.stdout: osd debug verify stray on activate = true 2026-02-20T21:02:50.402 INFO:teuthology.orchestra.run.trial196.stdout: osd debug trim objects = true 2026-02-20T21:02:50.402 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.402 INFO:teuthology.orchestra.run.trial196.stdout: osd open classes on start = true 2026-02-20T21:02:50.402 INFO:teuthology.orchestra.run.trial196.stdout: osd debug pg log writeout = true 2026-02-20T21:02:50.402 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.402 INFO:teuthology.orchestra.run.trial196.stdout: osd deep scrub update digest min age = 30 2026-02-20T21:02:50.402 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.402 INFO:teuthology.orchestra.run.trial196.stdout: osd map max advance = 10 2026-02-20T21:02:50.402 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.403 INFO:teuthology.orchestra.run.trial196.stdout: journal zero on create = true 2026-02-20T21:02:50.403 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.403 INFO:teuthology.orchestra.run.trial196.stdout: filestore ondisk finisher threads = 3 2026-02-20T21:02:50.403 INFO:teuthology.orchestra.run.trial196.stdout: filestore apply finisher threads = 3 2026-02-20T21:02:50.403 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.403 INFO:teuthology.orchestra.run.trial196.stdout: bdev debug aio = true 2026-02-20T21:02:50.403 INFO:teuthology.orchestra.run.trial196.stdout: osd debug misdirected ops = true 2026-02-20T21:02:50.403 INFO:teuthology.orchestra.run.trial196.stdout: bluestore block size = 96636764160 2026-02-20T21:02:50.403 INFO:teuthology.orchestra.run.trial196.stdout: bluestore compression algorithm = snappy 2026-02-20T21:02:50.403 INFO:teuthology.orchestra.run.trial196.stdout: bluestore compression mode = aggressive 2026-02-20T21:02:50.403 INFO:teuthology.orchestra.run.trial196.stdout: bluestore fsck on mount = True 2026-02-20T21:02:50.403 INFO:teuthology.orchestra.run.trial196.stdout: debug bluefs = 1/20 2026-02-20T21:02:50.404 INFO:teuthology.orchestra.run.trial196.stdout: debug bluestore = 1/20 2026-02-20T21:02:50.404 INFO:teuthology.orchestra.run.trial196.stdout: debug ms = 1 2026-02-20T21:02:50.404 INFO:teuthology.orchestra.run.trial196.stdout: debug osd = 20 2026-02-20T21:02:50.404 INFO:teuthology.orchestra.run.trial196.stdout: debug rocksdb = 4/10 2026-02-20T21:02:50.404 INFO:teuthology.orchestra.run.trial196.stdout: mon osd backfillfull_ratio = 0.85 2026-02-20T21:02:50.404 INFO:teuthology.orchestra.run.trial196.stdout: mon osd full ratio = 0.9 2026-02-20T21:02:50.404 INFO:teuthology.orchestra.run.trial196.stdout: mon osd nearfull ratio = 0.8 2026-02-20T21:02:50.404 INFO:teuthology.orchestra.run.trial196.stdout: osd failsafe full ratio = 0.95 2026-02-20T21:02:50.404 INFO:teuthology.orchestra.run.trial196.stdout: osd objectstore = bluestore 2026-02-20T21:02:50.404 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.404 INFO:teuthology.orchestra.run.trial196.stdout:[mgr] 2026-02-20T21:02:50.404 INFO:teuthology.orchestra.run.trial196.stdout: debug ms = 1 2026-02-20T21:02:50.404 INFO:teuthology.orchestra.run.trial196.stdout: debug mgr = 20 2026-02-20T21:02:50.405 INFO:teuthology.orchestra.run.trial196.stdout: debug mon = 20 2026-02-20T21:02:50.405 INFO:teuthology.orchestra.run.trial196.stdout: debug auth = 20 2026-02-20T21:02:50.405 INFO:teuthology.orchestra.run.trial196.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:02:50.405 INFO:teuthology.orchestra.run.trial196.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:02:50.405 INFO:teuthology.orchestra.run.trial196.stdout: mgr/telemetry/nag = false 2026-02-20T21:02:50.405 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.405 INFO:teuthology.orchestra.run.trial196.stdout:[mon] 2026-02-20T21:02:50.405 INFO:teuthology.orchestra.run.trial196.stdout: debug ms = 1 2026-02-20T21:02:50.405 INFO:teuthology.orchestra.run.trial196.stdout: debug mon = 20 2026-02-20T21:02:50.405 INFO:teuthology.orchestra.run.trial196.stdout: debug paxos = 20 2026-02-20T21:02:50.405 INFO:teuthology.orchestra.run.trial196.stdout: debug auth = 20 2026-02-20T21:02:50.405 INFO:teuthology.orchestra.run.trial196.stdout: mon data avail warn = 5 2026-02-20T21:02:50.406 INFO:teuthology.orchestra.run.trial196.stdout: mon mgr mkfs grace = 240 2026-02-20T21:02:50.406 INFO:teuthology.orchestra.run.trial196.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:02:50.406 INFO:teuthology.orchestra.run.trial196.stdout: mon osd reporter subtree level = osd 2026-02-20T21:02:50.406 INFO:teuthology.orchestra.run.trial196.stdout: mon osd prime pg temp = true 2026-02-20T21:02:50.406 INFO:teuthology.orchestra.run.trial196.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:02:50.406 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.406 INFO:teuthology.orchestra.run.trial196.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-20T21:02:50.406 INFO:teuthology.orchestra.run.trial196.stdout: auth mon ticket ttl = 660 # 11m 2026-02-20T21:02:50.406 INFO:teuthology.orchestra.run.trial196.stdout: auth service ticket ttl = 240 # 4m 2026-02-20T21:02:50.406 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.406 INFO:teuthology.orchestra.run.trial196.stdout: # don't complain about insecure global_id in the test suite 2026-02-20T21:02:50.406 INFO:teuthology.orchestra.run.trial196.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-20T21:02:50.407 INFO:teuthology.orchestra.run.trial196.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-20T21:02:50.407 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.407 INFO:teuthology.orchestra.run.trial196.stdout: # 1m isn't quite enough 2026-02-20T21:02:50.407 INFO:teuthology.orchestra.run.trial196.stdout: mon_down_mkfs_grace = 2m 2026-02-20T21:02:50.407 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.407 INFO:teuthology.orchestra.run.trial196.stdout: mon_warn_on_filestore_osds = false 2026-02-20T21:02:50.407 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:02:50.407 INFO:teuthology.orchestra.run.trial196.stdout:[client] 2026-02-20T21:02:50.407 INFO:teuthology.orchestra.run.trial196.stdout: rgw cache enabled = true 2026-02-20T21:02:50.407 INFO:teuthology.orchestra.run.trial196.stdout: rgw enable ops log = true 2026-02-20T21:02:50.407 INFO:teuthology.orchestra.run.trial196.stdout: rgw enable usage log = true 2026-02-20T21:02:50.407 INFO:teuthology.orchestra.run.trial196.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-20T21:02:50.407 INFO:teuthology.orchestra.run.trial196.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-20T21:02:50.408 INFO:teuthology.orchestra.run.trial196.stdout: debug rbd = 20 2026-02-20T21:02:50.408 INFO:teuthology.orchestra.run.trial196.stdout: debug rbd_mirror = 15 2026-02-20T21:02:50.408 INFO:teuthology.orchestra.run.trial196.stdout: log to stderr = False 2026-02-20T21:02:50.408 INFO:teuthology.orchestra.run.trial196.stdout: rbd default features = 125 2026-02-20T21:02:50.408 INFO:teuthology.orchestra.run.trial196.stdout:[mon.a] 2026-02-20T21:02:50.408 INFO:teuthology.orchestra.run.trial196.stdout:[client.mirror.0] 2026-02-20T21:02:50.408 INFO:teuthology.orchestra.run.trial196.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:50.408 INFO:teuthology.orchestra.run.trial196.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:50.408 INFO:teuthology.orchestra.run.trial196.stdout:[client.mirror.1] 2026-02-20T21:02:50.408 INFO:teuthology.orchestra.run.trial196.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:50.408 INFO:teuthology.orchestra.run.trial196.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:50.408 INFO:teuthology.orchestra.run.trial196.stdout:[client.mirror.2] 2026-02-20T21:02:50.409 INFO:teuthology.orchestra.run.trial196.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:50.409 INFO:teuthology.orchestra.run.trial196.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:50.409 INFO:teuthology.orchestra.run.trial196.stdout:[client.mirror.3] 2026-02-20T21:02:50.409 INFO:teuthology.orchestra.run.trial196.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:50.409 INFO:teuthology.orchestra.run.trial196.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:50.409 INFO:teuthology.orchestra.run.trial196.stdout:[client.mirror.4] 2026-02-20T21:02:50.409 INFO:teuthology.orchestra.run.trial196.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:50.409 INFO:teuthology.orchestra.run.trial196.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:50.409 INFO:teuthology.orchestra.run.trial196.stdout:[client.mirror.5] 2026-02-20T21:02:50.409 INFO:teuthology.orchestra.run.trial196.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:50.409 INFO:teuthology.orchestra.run.trial196.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:50.409 INFO:teuthology.orchestra.run.trial196.stdout:[client.mirror.6] 2026-02-20T21:02:50.409 INFO:teuthology.orchestra.run.trial196.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:50.410 INFO:teuthology.orchestra.run.trial196.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:50.435 INFO:teuthology.orchestra.run.trial199.stdout:[global] 2026-02-20T21:02:50.435 INFO:teuthology.orchestra.run.trial199.stdout: chdir = "" 2026-02-20T21:02:50.435 INFO:teuthology.orchestra.run.trial199.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-20T21:02:50.435 INFO:teuthology.orchestra.run.trial199.stdout: auth supported = cephx 2026-02-20T21:02:50.435 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.435 INFO:teuthology.orchestra.run.trial199.stdout: filestore xattr use omap = true 2026-02-20T21:02:50.435 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.435 INFO:teuthology.orchestra.run.trial199.stdout: mon clock drift allowed = 1.000 2026-02-20T21:02:50.436 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.436 INFO:teuthology.orchestra.run.trial199.stdout: osd crush chooseleaf type = 0 2026-02-20T21:02:50.436 INFO:teuthology.orchestra.run.trial199.stdout: auth debug = true 2026-02-20T21:02:50.436 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.436 INFO:teuthology.orchestra.run.trial199.stdout: ms die on old message = true 2026-02-20T21:02:50.436 INFO:teuthology.orchestra.run.trial199.stdout: ms die on bug = true 2026-02-20T21:02:50.436 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.436 INFO:teuthology.orchestra.run.trial199.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-20T21:02:50.436 INFO:teuthology.orchestra.run.trial199.stdout: mon pg warn max object skew = 0 2026-02-20T21:02:50.436 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.436 INFO:teuthology.orchestra.run.trial199.stdout: # disable pg_autoscaler by default for new pools 2026-02-20T21:02:50.436 INFO:teuthology.orchestra.run.trial199.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-20T21:02:50.436 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.437 INFO:teuthology.orchestra.run.trial199.stdout: osd pool default size = 2 2026-02-20T21:02:50.437 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.437 INFO:teuthology.orchestra.run.trial199.stdout: mon osd allow primary affinity = true 2026-02-20T21:02:50.437 INFO:teuthology.orchestra.run.trial199.stdout: mon osd allow pg remap = true 2026-02-20T21:02:50.437 INFO:teuthology.orchestra.run.trial199.stdout: mon warn on legacy crush tunables = false 2026-02-20T21:02:50.437 INFO:teuthology.orchestra.run.trial199.stdout: mon warn on crush straw calc version zero = false 2026-02-20T21:02:50.437 INFO:teuthology.orchestra.run.trial199.stdout: mon warn on no sortbitwise = false 2026-02-20T21:02:50.437 INFO:teuthology.orchestra.run.trial199.stdout: mon warn on osd down out interval zero = false 2026-02-20T21:02:50.437 INFO:teuthology.orchestra.run.trial199.stdout: mon warn on too few osds = false 2026-02-20T21:02:50.437 INFO:teuthology.orchestra.run.trial199.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-20T21:02:50.437 INFO:teuthology.orchestra.run.trial199.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-20T21:02:50.437 INFO:teuthology.orchestra.run.trial199.stdout: mon_allow_pool_size_one = true 2026-02-20T21:02:50.438 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.438 INFO:teuthology.orchestra.run.trial199.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-20T21:02:50.438 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.438 INFO:teuthology.orchestra.run.trial199.stdout: osd default data pool replay window = 5 2026-02-20T21:02:50.438 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.438 INFO:teuthology.orchestra.run.trial199.stdout: mon allow pool delete = true 2026-02-20T21:02:50.438 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.438 INFO:teuthology.orchestra.run.trial199.stdout: mon cluster log file level = debug 2026-02-20T21:02:50.438 INFO:teuthology.orchestra.run.trial199.stdout: debug asserts on shutdown = true 2026-02-20T21:02:50.438 INFO:teuthology.orchestra.run.trial199.stdout: mon health detail to clog = false 2026-02-20T21:02:50.438 INFO:teuthology.orchestra.run.trial199.stdout: mon host = 10.20.193.199 2026-02-20T21:02:50.438 INFO:teuthology.orchestra.run.trial199.stdout: mon client directed command retry = 5 2026-02-20T21:02:50.439 INFO:teuthology.orchestra.run.trial199.stdout: mon warn on pool no app = False 2026-02-20T21:02:50.439 INFO:teuthology.orchestra.run.trial199.stdout: ms inject socket failures = 5000 2026-02-20T21:02:50.439 INFO:teuthology.orchestra.run.trial199.stdout: fsid = f480090c-6109-4036-a7fa-8e7cbf4099eb 2026-02-20T21:02:50.439 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.439 INFO:teuthology.orchestra.run.trial199.stdout:[osd] 2026-02-20T21:02:50.439 INFO:teuthology.orchestra.run.trial199.stdout: osd journal size = 100 2026-02-20T21:02:50.439 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.439 INFO:teuthology.orchestra.run.trial199.stdout: osd scrub load threshold = 5.0 2026-02-20T21:02:50.439 INFO:teuthology.orchestra.run.trial199.stdout: osd scrub max interval = 600 2026-02-20T21:02:50.440 INFO:teuthology.orchestra.run.trial199.stdout: osd mclock profile = high_recovery_ops 2026-02-20T21:02:50.440 INFO:teuthology.orchestra.run.trial199.stdout: osd mclock skip benchmark = true 2026-02-20T21:02:50.440 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.440 INFO:teuthology.orchestra.run.trial199.stdout: osd recover clone overlap = true 2026-02-20T21:02:50.440 INFO:teuthology.orchestra.run.trial199.stdout: osd recovery max chunk = 1048576 2026-02-20T21:02:50.440 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.440 INFO:teuthology.orchestra.run.trial199.stdout: osd debug shutdown = true 2026-02-20T21:02:50.440 INFO:teuthology.orchestra.run.trial199.stdout: osd debug op order = true 2026-02-20T21:02:50.440 INFO:teuthology.orchestra.run.trial199.stdout: osd debug verify stray on activate = true 2026-02-20T21:02:50.440 INFO:teuthology.orchestra.run.trial199.stdout: osd debug trim objects = true 2026-02-20T21:02:50.440 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.441 INFO:teuthology.orchestra.run.trial199.stdout: osd open classes on start = true 2026-02-20T21:02:50.441 INFO:teuthology.orchestra.run.trial199.stdout: osd debug pg log writeout = true 2026-02-20T21:02:50.441 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.441 INFO:teuthology.orchestra.run.trial199.stdout: osd deep scrub update digest min age = 30 2026-02-20T21:02:50.441 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.441 INFO:teuthology.orchestra.run.trial199.stdout: osd map max advance = 10 2026-02-20T21:02:50.441 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.441 INFO:teuthology.orchestra.run.trial199.stdout: journal zero on create = true 2026-02-20T21:02:50.441 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.441 INFO:teuthology.orchestra.run.trial199.stdout: filestore ondisk finisher threads = 3 2026-02-20T21:02:50.441 INFO:teuthology.orchestra.run.trial199.stdout: filestore apply finisher threads = 3 2026-02-20T21:02:50.441 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.442 INFO:teuthology.orchestra.run.trial199.stdout: bdev debug aio = true 2026-02-20T21:02:50.442 INFO:teuthology.orchestra.run.trial199.stdout: osd debug misdirected ops = true 2026-02-20T21:02:50.442 INFO:teuthology.orchestra.run.trial199.stdout: bluestore block size = 96636764160 2026-02-20T21:02:50.442 INFO:teuthology.orchestra.run.trial199.stdout: bluestore compression algorithm = snappy 2026-02-20T21:02:50.442 INFO:teuthology.orchestra.run.trial199.stdout: bluestore compression mode = aggressive 2026-02-20T21:02:50.442 INFO:teuthology.orchestra.run.trial199.stdout: bluestore fsck on mount = True 2026-02-20T21:02:50.442 INFO:teuthology.orchestra.run.trial199.stdout: debug bluefs = 1/20 2026-02-20T21:02:50.442 INFO:teuthology.orchestra.run.trial199.stdout: debug bluestore = 1/20 2026-02-20T21:02:50.442 INFO:teuthology.orchestra.run.trial199.stdout: debug ms = 1 2026-02-20T21:02:50.442 INFO:teuthology.orchestra.run.trial199.stdout: debug osd = 20 2026-02-20T21:02:50.442 INFO:teuthology.orchestra.run.trial199.stdout: debug rocksdb = 4/10 2026-02-20T21:02:50.442 INFO:teuthology.orchestra.run.trial199.stdout: mon osd backfillfull_ratio = 0.85 2026-02-20T21:02:50.442 INFO:teuthology.orchestra.run.trial199.stdout: mon osd full ratio = 0.9 2026-02-20T21:02:50.443 INFO:teuthology.orchestra.run.trial199.stdout: mon osd nearfull ratio = 0.8 2026-02-20T21:02:50.443 INFO:teuthology.orchestra.run.trial199.stdout: osd failsafe full ratio = 0.95 2026-02-20T21:02:50.443 INFO:teuthology.orchestra.run.trial199.stdout: osd objectstore = bluestore 2026-02-20T21:02:50.443 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.443 INFO:teuthology.orchestra.run.trial199.stdout:[mgr] 2026-02-20T21:02:50.443 INFO:teuthology.orchestra.run.trial199.stdout: debug ms = 1 2026-02-20T21:02:50.443 INFO:teuthology.orchestra.run.trial199.stdout: debug mgr = 20 2026-02-20T21:02:50.443 INFO:teuthology.orchestra.run.trial199.stdout: debug mon = 20 2026-02-20T21:02:50.443 INFO:teuthology.orchestra.run.trial199.stdout: debug auth = 20 2026-02-20T21:02:50.443 INFO:teuthology.orchestra.run.trial199.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:02:50.443 INFO:teuthology.orchestra.run.trial199.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:02:50.443 INFO:teuthology.orchestra.run.trial199.stdout: mgr/telemetry/nag = false 2026-02-20T21:02:50.443 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.444 INFO:teuthology.orchestra.run.trial199.stdout:[mon] 2026-02-20T21:02:50.444 INFO:teuthology.orchestra.run.trial199.stdout: debug ms = 1 2026-02-20T21:02:50.444 INFO:teuthology.orchestra.run.trial199.stdout: debug mon = 20 2026-02-20T21:02:50.444 INFO:teuthology.orchestra.run.trial199.stdout: debug paxos = 20 2026-02-20T21:02:50.444 INFO:teuthology.orchestra.run.trial199.stdout: debug auth = 20 2026-02-20T21:02:50.444 INFO:teuthology.orchestra.run.trial199.stdout: mon data avail warn = 5 2026-02-20T21:02:50.444 INFO:teuthology.orchestra.run.trial199.stdout: mon mgr mkfs grace = 240 2026-02-20T21:02:50.444 INFO:teuthology.orchestra.run.trial199.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:02:50.444 INFO:teuthology.orchestra.run.trial199.stdout: mon osd reporter subtree level = osd 2026-02-20T21:02:50.444 INFO:teuthology.orchestra.run.trial199.stdout: mon osd prime pg temp = true 2026-02-20T21:02:50.444 INFO:teuthology.orchestra.run.trial199.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:02:50.444 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.445 INFO:teuthology.orchestra.run.trial199.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-20T21:02:50.445 INFO:teuthology.orchestra.run.trial199.stdout: auth mon ticket ttl = 660 # 11m 2026-02-20T21:02:50.445 INFO:teuthology.orchestra.run.trial199.stdout: auth service ticket ttl = 240 # 4m 2026-02-20T21:02:50.445 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.445 INFO:teuthology.orchestra.run.trial199.stdout: # don't complain about insecure global_id in the test suite 2026-02-20T21:02:50.445 INFO:teuthology.orchestra.run.trial199.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-20T21:02:50.445 INFO:teuthology.orchestra.run.trial199.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-20T21:02:50.445 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.445 INFO:teuthology.orchestra.run.trial199.stdout: # 1m isn't quite enough 2026-02-20T21:02:50.445 INFO:teuthology.orchestra.run.trial199.stdout: mon_down_mkfs_grace = 2m 2026-02-20T21:02:50.445 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.445 INFO:teuthology.orchestra.run.trial199.stdout: mon_warn_on_filestore_osds = false 2026-02-20T21:02:50.446 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:50.446 INFO:teuthology.orchestra.run.trial199.stdout:[client] 2026-02-20T21:02:50.446 INFO:teuthology.orchestra.run.trial199.stdout: rgw cache enabled = true 2026-02-20T21:02:50.446 INFO:teuthology.orchestra.run.trial199.stdout: rgw enable ops log = true 2026-02-20T21:02:50.446 INFO:teuthology.orchestra.run.trial199.stdout: rgw enable usage log = true 2026-02-20T21:02:50.446 INFO:teuthology.orchestra.run.trial199.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-20T21:02:50.446 INFO:teuthology.orchestra.run.trial199.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-20T21:02:50.446 INFO:teuthology.orchestra.run.trial199.stdout: debug rbd = 20 2026-02-20T21:02:50.446 INFO:teuthology.orchestra.run.trial199.stdout: debug rbd_mirror = 15 2026-02-20T21:02:50.446 INFO:teuthology.orchestra.run.trial199.stdout: log to stderr = False 2026-02-20T21:02:50.446 INFO:teuthology.orchestra.run.trial199.stdout: rbd default features = 125 2026-02-20T21:02:50.446 INFO:teuthology.orchestra.run.trial199.stdout:[mon.a] 2026-02-20T21:02:50.446 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.0] 2026-02-20T21:02:50.447 INFO:teuthology.orchestra.run.trial199.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:50.447 INFO:teuthology.orchestra.run.trial199.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:50.447 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.1] 2026-02-20T21:02:50.447 INFO:teuthology.orchestra.run.trial199.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:50.447 INFO:teuthology.orchestra.run.trial199.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:50.447 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.2] 2026-02-20T21:02:50.447 INFO:teuthology.orchestra.run.trial199.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:50.447 INFO:teuthology.orchestra.run.trial199.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:50.447 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.3] 2026-02-20T21:02:50.447 INFO:teuthology.orchestra.run.trial199.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:50.447 INFO:teuthology.orchestra.run.trial199.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:50.447 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.4] 2026-02-20T21:02:50.448 INFO:teuthology.orchestra.run.trial199.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:50.448 INFO:teuthology.orchestra.run.trial199.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:50.448 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.5] 2026-02-20T21:02:50.448 INFO:teuthology.orchestra.run.trial199.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:50.448 INFO:teuthology.orchestra.run.trial199.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:50.448 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.6] 2026-02-20T21:02:50.448 INFO:teuthology.orchestra.run.trial199.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:02:50.448 INFO:teuthology.orchestra.run.trial199.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:02:50.449 INFO:tasks.ceph:Creating admin key on cluster2.mon.a... 2026-02-20T21:02:50.450 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster2.keyring 2026-02-20T21:02:50.532 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-20T21:02:50.532 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:50.532 DEBUG:teuthology.orchestra.run.trial199:> dd if=/etc/ceph/cluster2.keyring of=/dev/stdout 2026-02-20T21:02:50.588 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:50.589 DEBUG:teuthology.orchestra.run.trial199:> dd if=/home/ubuntu/cephtest/cluster2.monmap of=/dev/stdout 2026-02-20T21:02:50.645 INFO:tasks.ceph:Sending monmap to node ubuntu@trial196.front.sepia.ceph.com 2026-02-20T21:02:50.646 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T21:02:50.646 DEBUG:teuthology.orchestra.run.trial196:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-20T21:02:50.646 DEBUG:teuthology.orchestra.run.trial196:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-20T21:02:50.680 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T21:02:50.680 DEBUG:teuthology.orchestra.run.trial196:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:02:50.738 INFO:tasks.ceph:Sending monmap to node ubuntu@trial199.front.sepia.ceph.com 2026-02-20T21:02:50.738 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:50.738 DEBUG:teuthology.orchestra.run.trial199:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-20T21:02:50.738 DEBUG:teuthology.orchestra.run.trial199:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-20T21:02:50.774 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:50.774 DEBUG:teuthology.orchestra.run.trial199:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:02:50.831 INFO:tasks.ceph:Setting up mon nodes... 2026-02-20T21:02:50.832 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-20T21:02:50.832 DEBUG:teuthology.orchestra.run.trial196:> sudo mkdir -p /var/lib/ceph/mgr/cluster2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-20T21:02:50.875 INFO:teuthology.orchestra.run.trial196.stdout:creating /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-20T21:02:50.878 INFO:tasks.ceph:Setting up mds nodes... 2026-02-20T21:02:50.879 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-20T21:02:50.879 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster2.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.0.keyring 2026-02-20T21:02:50.912 INFO:teuthology.orchestra.run.trial196.stdout:creating /etc/ceph/cluster2.client.0.keyring 2026-02-20T21:02:50.923 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster2.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.keyring 2026-02-20T21:02:50.960 INFO:teuthology.orchestra.run.trial199.stdout:creating /etc/ceph/cluster2.client.mirror.keyring 2026-02-20T21:02:50.971 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster2.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-20T21:02:51.046 INFO:teuthology.orchestra.run.trial199.stdout:creating /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-20T21:02:51.057 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster2.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-20T21:02:51.134 INFO:teuthology.orchestra.run.trial199.stdout:creating /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-20T21:02:51.146 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster2.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-20T21:02:51.221 INFO:teuthology.orchestra.run.trial199.stdout:creating /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-20T21:02:51.232 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster2.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-20T21:02:51.306 INFO:teuthology.orchestra.run.trial199.stdout:creating /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-20T21:02:51.317 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster2.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-20T21:02:51.392 INFO:teuthology.orchestra.run.trial199.stdout:creating /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-20T21:02:51.402 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster2.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-20T21:02:51.477 INFO:teuthology.orchestra.run.trial199.stdout:creating /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-20T21:02:51.488 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster2.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-20T21:02:51.561 INFO:teuthology.orchestra.run.trial199.stdout:creating /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-20T21:02:51.571 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-20T21:02:51.571 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial196.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@trial199.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-20T21:02:51.572 DEBUG:teuthology.orchestra.run.trial199:> sudo mkdir -p /var/lib/ceph/osd/cluster2-0 2026-02-20T21:02:51.636 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:02:51.636 INFO:tasks.ceph:role: cluster2.osd.0 2026-02-20T21:02:51.636 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial199.front.sepia.ceph.com 2026-02-20T21:02:51.636 DEBUG:teuthology.orchestra.run.trial199:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-20T21:02:51.700 INFO:teuthology.orchestra.run.trial199.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-20T21:02:51.700 INFO:teuthology.orchestra.run.trial199.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-20T21:02:51.700 INFO:teuthology.orchestra.run.trial199.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-20T21:02:51.700 INFO:teuthology.orchestra.run.trial199.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-20T21:02:51.700 INFO:teuthology.orchestra.run.trial199.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-20T21:02:51.700 INFO:teuthology.orchestra.run.trial199.stdout: = sunit=0 swidth=0 blks 2026-02-20T21:02:51.700 INFO:teuthology.orchestra.run.trial199.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-20T21:02:51.700 INFO:teuthology.orchestra.run.trial199.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-20T21:02:51.700 INFO:teuthology.orchestra.run.trial199.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-20T21:02:51.700 INFO:teuthology.orchestra.run.trial199.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-20T21:02:52.304 INFO:teuthology.orchestra.run.trial199.stdout:Discarding blocks...Done. 2026-02-20T21:02:52.306 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial199.front.sepia.ceph.com -o noatime 2026-02-20T21:02:52.307 DEBUG:teuthology.orchestra.run.trial199:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster2-0 2026-02-20T21:02:52.374 DEBUG:teuthology.orchestra.run.trial199:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-0 2026-02-20T21:02:52.442 DEBUG:teuthology.orchestra.run.trial199:> sudo mkdir -p /var/lib/ceph/osd/cluster2-1 2026-02-20T21:02:52.506 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:02:52.506 INFO:tasks.ceph:role: cluster2.osd.1 2026-02-20T21:02:52.506 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial199.front.sepia.ceph.com 2026-02-20T21:02:52.506 DEBUG:teuthology.orchestra.run.trial199:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-20T21:02:52.571 INFO:teuthology.orchestra.run.trial199.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-20T21:02:52.571 INFO:teuthology.orchestra.run.trial199.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-20T21:02:52.571 INFO:teuthology.orchestra.run.trial199.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-20T21:02:52.571 INFO:teuthology.orchestra.run.trial199.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-20T21:02:52.571 INFO:teuthology.orchestra.run.trial199.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-20T21:02:52.571 INFO:teuthology.orchestra.run.trial199.stdout: = sunit=0 swidth=0 blks 2026-02-20T21:02:52.571 INFO:teuthology.orchestra.run.trial199.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-20T21:02:52.571 INFO:teuthology.orchestra.run.trial199.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-20T21:02:52.571 INFO:teuthology.orchestra.run.trial199.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-20T21:02:52.571 INFO:teuthology.orchestra.run.trial199.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-20T21:02:53.143 INFO:teuthology.orchestra.run.trial199.stdout:Discarding blocks...Done. 2026-02-20T21:02:53.145 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial199.front.sepia.ceph.com -o noatime 2026-02-20T21:02:53.145 DEBUG:teuthology.orchestra.run.trial199:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster2-1 2026-02-20T21:02:53.212 DEBUG:teuthology.orchestra.run.trial199:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-1 2026-02-20T21:02:53.279 DEBUG:teuthology.orchestra.run.trial199:> sudo mkdir -p /var/lib/ceph/osd/cluster2-2 2026-02-20T21:02:53.344 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:02:53.344 INFO:tasks.ceph:role: cluster2.osd.2 2026-02-20T21:02:53.344 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial199.front.sepia.ceph.com 2026-02-20T21:02:53.345 DEBUG:teuthology.orchestra.run.trial199:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-20T21:02:53.408 INFO:teuthology.orchestra.run.trial199.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-20T21:02:53.408 INFO:teuthology.orchestra.run.trial199.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-20T21:02:53.409 INFO:teuthology.orchestra.run.trial199.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-20T21:02:53.409 INFO:teuthology.orchestra.run.trial199.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-20T21:02:53.409 INFO:teuthology.orchestra.run.trial199.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-20T21:02:53.409 INFO:teuthology.orchestra.run.trial199.stdout: = sunit=0 swidth=0 blks 2026-02-20T21:02:53.409 INFO:teuthology.orchestra.run.trial199.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-20T21:02:53.409 INFO:teuthology.orchestra.run.trial199.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-20T21:02:53.409 INFO:teuthology.orchestra.run.trial199.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-20T21:02:53.409 INFO:teuthology.orchestra.run.trial199.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-20T21:02:53.998 INFO:teuthology.orchestra.run.trial199.stdout:Discarding blocks...Done. 2026-02-20T21:02:54.000 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial199.front.sepia.ceph.com -o noatime 2026-02-20T21:02:54.001 DEBUG:teuthology.orchestra.run.trial199:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster2-2 2026-02-20T21:02:54.068 DEBUG:teuthology.orchestra.run.trial199:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-2 2026-02-20T21:02:54.136 DEBUG:teuthology.orchestra.run.trial199:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:02:54.213 INFO:teuthology.orchestra.run.trial199.stderr:2026-02-20T21:02:54.210+0000 7f5c3b149900 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-0/keyring: can't open /var/lib/ceph/osd/cluster2-0/keyring: (2) No such file or directory 2026-02-20T21:02:54.213 INFO:teuthology.orchestra.run.trial199.stderr:2026-02-20T21:02:54.211+0000 7f5c3b149900 -1 created new key in keyring /var/lib/ceph/osd/cluster2-0/keyring 2026-02-20T21:02:54.213 INFO:teuthology.orchestra.run.trial199.stderr:2026-02-20T21:02:54.211+0000 7f5c3b149900 -1 bdev(0x55d52f7c1800 /var/lib/ceph/osd/cluster2-0/block) open stat got: (1) Operation not permitted 2026-02-20T21:02:54.213 INFO:teuthology.orchestra.run.trial199.stderr:2026-02-20T21:02:54.211+0000 7f5c3b149900 -1 bluestore(/var/lib/ceph/osd/cluster2-0) _read_fsid unparsable uuid 2026-02-20T21:02:54.664 DEBUG:teuthology.orchestra.run.trial199:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-0 2026-02-20T21:02:54.731 DEBUG:teuthology.orchestra.run.trial199:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:02:54.810 INFO:teuthology.orchestra.run.trial199.stderr:2026-02-20T21:02:54.807+0000 7f8213ef7900 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-1/keyring: can't open /var/lib/ceph/osd/cluster2-1/keyring: (2) No such file or directory 2026-02-20T21:02:54.810 INFO:teuthology.orchestra.run.trial199.stderr:2026-02-20T21:02:54.807+0000 7f8213ef7900 -1 created new key in keyring /var/lib/ceph/osd/cluster2-1/keyring 2026-02-20T21:02:54.810 INFO:teuthology.orchestra.run.trial199.stderr:2026-02-20T21:02:54.807+0000 7f8213ef7900 -1 bdev(0x55cb5b1e5800 /var/lib/ceph/osd/cluster2-1/block) open stat got: (1) Operation not permitted 2026-02-20T21:02:54.810 INFO:teuthology.orchestra.run.trial199.stderr:2026-02-20T21:02:54.807+0000 7f8213ef7900 -1 bluestore(/var/lib/ceph/osd/cluster2-1) _read_fsid unparsable uuid 2026-02-20T21:02:55.256 DEBUG:teuthology.orchestra.run.trial199:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-1 2026-02-20T21:02:55.323 DEBUG:teuthology.orchestra.run.trial199:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:02:55.401 INFO:teuthology.orchestra.run.trial199.stderr:2026-02-20T21:02:55.398+0000 7fda02eba900 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-2/keyring: can't open /var/lib/ceph/osd/cluster2-2/keyring: (2) No such file or directory 2026-02-20T21:02:55.402 INFO:teuthology.orchestra.run.trial199.stderr:2026-02-20T21:02:55.399+0000 7fda02eba900 -1 created new key in keyring /var/lib/ceph/osd/cluster2-2/keyring 2026-02-20T21:02:55.402 INFO:teuthology.orchestra.run.trial199.stderr:2026-02-20T21:02:55.399+0000 7fda02eba900 -1 bdev(0x55f355d8b800 /var/lib/ceph/osd/cluster2-2/block) open stat got: (1) Operation not permitted 2026-02-20T21:02:55.402 INFO:teuthology.orchestra.run.trial199.stderr:2026-02-20T21:02:55.399+0000 7fda02eba900 -1 bluestore(/var/lib/ceph/osd/cluster2-2) _read_fsid unparsable uuid 2026-02-20T21:02:55.865 DEBUG:teuthology.orchestra.run.trial199:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-2 2026-02-20T21:02:55.932 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-20T21:02:55.932 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T21:02:55.932 DEBUG:teuthology.orchestra.run.trial196:> sudo dd if=/var/lib/ceph/mgr/cluster2-x/keyring of=/dev/stdout 2026-02-20T21:02:55.958 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:55.959 DEBUG:teuthology.orchestra.run.trial199:> sudo dd if=/var/lib/ceph/osd/cluster2-0/keyring of=/dev/stdout 2026-02-20T21:02:55.995 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:55.996 DEBUG:teuthology.orchestra.run.trial199:> sudo dd if=/var/lib/ceph/osd/cluster2-1/keyring of=/dev/stdout 2026-02-20T21:02:56.060 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:56.060 DEBUG:teuthology.orchestra.run.trial199:> sudo dd if=/var/lib/ceph/osd/cluster2-2/keyring of=/dev/stdout 2026-02-20T21:02:56.125 DEBUG:teuthology.orchestra.run.trial196:> set -ex 2026-02-20T21:02:56.125 DEBUG:teuthology.orchestra.run.trial196:> dd if=/etc/ceph/cluster2.client.0.keyring of=/dev/stdout 2026-02-20T21:02:56.143 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:56.144 DEBUG:teuthology.orchestra.run.trial199:> dd if=/etc/ceph/cluster2.client.mirror.keyring of=/dev/stdout 2026-02-20T21:02:56.182 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:56.182 DEBUG:teuthology.orchestra.run.trial199:> dd if=/etc/ceph/cluster2.client.mirror.0.keyring of=/dev/stdout 2026-02-20T21:02:56.239 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:56.239 DEBUG:teuthology.orchestra.run.trial199:> dd if=/etc/ceph/cluster2.client.mirror.1.keyring of=/dev/stdout 2026-02-20T21:02:56.295 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:56.295 DEBUG:teuthology.orchestra.run.trial199:> dd if=/etc/ceph/cluster2.client.mirror.2.keyring of=/dev/stdout 2026-02-20T21:02:56.351 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:56.351 DEBUG:teuthology.orchestra.run.trial199:> dd if=/etc/ceph/cluster2.client.mirror.3.keyring of=/dev/stdout 2026-02-20T21:02:56.408 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:56.408 DEBUG:teuthology.orchestra.run.trial199:> dd if=/etc/ceph/cluster2.client.mirror.4.keyring of=/dev/stdout 2026-02-20T21:02:56.465 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:56.465 DEBUG:teuthology.orchestra.run.trial199:> dd if=/etc/ceph/cluster2.client.mirror.5.keyring of=/dev/stdout 2026-02-20T21:02:56.521 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:56.521 DEBUG:teuthology.orchestra.run.trial199:> dd if=/etc/ceph/cluster2.client.mirror.6.keyring of=/dev/stdout 2026-02-20T21:02:56.577 INFO:tasks.ceph:Adding keys to all mons... 2026-02-20T21:02:56.578 DEBUG:teuthology.orchestra.run.trial199:> sudo tee -a /etc/ceph/cluster2.keyring 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout:[mgr.x] 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout: key = AQD6y5hpy1geNBAAG7LpGqF3P9ON8VfkhxN8vA== 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout:[osd.0] 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout: key = AQD+y5hpvHahDBAAaUaOs1QwztD8q8N5gNGphA== 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout:[osd.1] 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout: key = AQD+y5hpBDYxMBAACqCrdWqh0pGdQ6L6/4ytfw== 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout:[osd.2] 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout: key = AQD/y5hpfW7YFxAAQF2o5woL67JYsbA5LBcToQ== 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout:[client.0] 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout: key = AQD6y5hpFcdRNhAA6VnB7pncdeA4NNbsEs7r1g== 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror] 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout: key = AQD6y5hpbKErORAAxF/VX2dLPA47r7ykjtzV2Q== 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.0] 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout: key = AQD7y5hpz2aqAhAARIW7sEk5C4HBhYBtw4xe2Q== 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.1] 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout: key = AQD7y5hpiF7sBxAAQ8ndp7vJ7QJN9ZsxRdB1IQ== 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.2] 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout: key = AQD7y5hpjAkZDRAAIWy2xbLePOOnOK1o/SrTWw== 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.3] 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout: key = AQD7y5hpZVUvEhAA7EBX09mGGZujE/FsO2e8XA== 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.4] 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout: key = AQD7y5hpLIFEFxAA1+wNdmlvRMvGOz0bRm7+ug== 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.5] 2026-02-20T21:02:56.640 INFO:teuthology.orchestra.run.trial199.stdout: key = AQD7y5hpISFWHBAAHrUnn8ssEvp6oL69w8Nd8g== 2026-02-20T21:02:56.641 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.6] 2026-02-20T21:02:56.641 INFO:teuthology.orchestra.run.trial199.stdout: key = AQD7y5hp7rJYIRAAL0Q5kIzSqnIGA6lgpx28HA== 2026-02-20T21:02:56.641 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-20T21:02:56.721 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-20T21:02:56.802 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-20T21:02:56.883 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-20T21:02:56.962 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:02:57.044 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:02:57.125 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:02:57.208 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:02:57.288 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:02:57.370 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:02:57.450 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:02:57.532 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:02:57.613 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:02:57.695 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-20T21:02:57.695 DEBUG:teuthology.orchestra.run.trial199:> sudo mkdir -p /var/lib/ceph/mon/cluster2-a 2026-02-20T21:02:57.759 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster2 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster2.monmap --keyring /etc/ceph/cluster2.keyring 2026-02-20T21:02:57.845 DEBUG:teuthology.orchestra.run.trial199:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster2-a 2026-02-20T21:02:57.869 DEBUG:teuthology.orchestra.run.trial199:> rm -- /home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:02:57.925 INFO:tasks.ceph:Starting mon daemons in cluster cluster2... 2026-02-20T21:02:57.926 INFO:tasks.ceph.cluster2.mon.a:Restarting daemon 2026-02-20T21:02:57.926 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster2 -i a 2026-02-20T21:02:57.968 INFO:tasks.ceph.cluster2.mon.a:Started 2026-02-20T21:02:57.968 INFO:tasks.ceph:Starting mgr daemons in cluster cluster2... 2026-02-20T21:02:57.969 INFO:tasks.ceph.cluster2.mgr.x:Restarting daemon 2026-02-20T21:02:57.969 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster2 -i x 2026-02-20T21:02:57.972 INFO:tasks.ceph.cluster2.mgr.x:Started 2026-02-20T21:02:57.972 DEBUG:tasks.ceph:set 0 configs 2026-02-20T21:02:57.972 DEBUG:teuthology.orchestra.run.trial199:> sudo ceph --cluster cluster2 config dump 2026-02-20T21:02:58.070 INFO:teuthology.orchestra.run.trial199.stdout:WHO MASK LEVEL OPTION VALUE RO 2026-02-20T21:02:58.077 INFO:tasks.ceph:Setting crush tunables to default 2026-02-20T21:02:58.077 DEBUG:teuthology.orchestra.run.trial199:> sudo ceph --cluster cluster2 osd crush tunables default 2026-02-20T21:02:58.176 INFO:teuthology.orchestra.run.trial199.stderr:adjusted tunables profile to default 2026-02-20T21:02:58.183 INFO:tasks.ceph:check_enable_crimson: False 2026-02-20T21:02:58.183 INFO:tasks.ceph:Starting osd daemons in cluster cluster2... 2026-02-20T21:02:58.184 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:58.184 DEBUG:teuthology.orchestra.run.trial199:> sudo dd if=/var/lib/ceph/osd/cluster2-0/fsid of=/dev/stdout 2026-02-20T21:02:58.208 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:58.209 DEBUG:teuthology.orchestra.run.trial199:> sudo dd if=/var/lib/ceph/osd/cluster2-1/fsid of=/dev/stdout 2026-02-20T21:02:58.272 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:02:58.272 DEBUG:teuthology.orchestra.run.trial199:> sudo dd if=/var/lib/ceph/osd/cluster2-2/fsid of=/dev/stdout 2026-02-20T21:02:58.337 DEBUG:teuthology.orchestra.run.trial199:> sudo ceph --cluster cluster2 osd new ea3bd762-09f9-4bf7-b187-ff293d2e7ade 0 2026-02-20T21:02:58.480 INFO:teuthology.orchestra.run.trial199.stdout:0 2026-02-20T21:02:58.487 DEBUG:teuthology.orchestra.run.trial199:> sudo ceph --cluster cluster2 osd new 7b93c7a9-2b3d-4232-9811-774ad9478d1b 1 2026-02-20T21:02:58.588 INFO:teuthology.orchestra.run.trial199.stdout:1 2026-02-20T21:02:58.596 DEBUG:teuthology.orchestra.run.trial199:> sudo ceph --cluster cluster2 osd new 13b69e07-8c48-41b2-bd6a-7ffe80cec20b 2 2026-02-20T21:02:58.698 INFO:teuthology.orchestra.run.trial199.stdout:2 2026-02-20T21:02:58.705 INFO:tasks.ceph.cluster2.osd.0:Restarting daemon 2026-02-20T21:02:58.706 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 0 2026-02-20T21:02:58.707 INFO:tasks.ceph.cluster2.mgr.x.trial196.stderr:/usr/lib64/python3.9/site-packages/scipy/__init__.py:73: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-20T21:02:58.707 INFO:tasks.ceph.cluster2.mgr.x.trial196.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-20T21:02:58.707 INFO:tasks.ceph.cluster2.mgr.x.trial196.stderr: from numpy import show_config as show_numpy_config 2026-02-20T21:02:58.708 INFO:tasks.ceph.cluster2.osd.0:Started 2026-02-20T21:02:58.708 INFO:tasks.ceph.cluster2.osd.1:Restarting daemon 2026-02-20T21:02:58.708 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 1 2026-02-20T21:02:58.711 INFO:tasks.ceph.cluster2.osd.1:Started 2026-02-20T21:02:58.711 INFO:tasks.ceph.cluster2.osd.2:Restarting daemon 2026-02-20T21:02:58.711 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 2 2026-02-20T21:02:58.714 INFO:tasks.ceph.cluster2.osd.2:Started 2026-02-20T21:02:58.715 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-20T21:02:58.816 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:02:58.816 INFO:teuthology.orchestra.run.trial199.stdout:{"epoch":5,"fsid":"f480090c-6109-4036-a7fa-8e7cbf4099eb","created":"2026-02-20T21:02:58.028370+0000","modified":"2026-02-20T21:02:58.696055+0000","last_up_change":"0.000000","last_in_change":"2026-02-20T21:02:58.696055+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"ea3bd762-09f9-4bf7-b187-ff293d2e7ade","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"7b93c7a9-2b3d-4232-9811-774ad9478d1b","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"13b69e07-8c48-41b2-bd6a-7ffe80cec20b","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:02:58.823 INFO:tasks.ceph.ceph_manager.cluster2:[] 2026-02-20T21:02:58.823 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-20T21:02:58.945 INFO:tasks.ceph.cluster2.osd.1.trial199.stderr:2026-02-20T21:02:58.942+0000 7fefb8dc9900 -1 Falling back to public interface 2026-02-20T21:02:58.948 INFO:tasks.ceph.cluster2.osd.0.trial199.stderr:2026-02-20T21:02:58.945+0000 7f16b001c900 -1 Falling back to public interface 2026-02-20T21:02:58.949 INFO:tasks.ceph.cluster2.osd.2.trial199.stderr:2026-02-20T21:02:58.946+0000 7fbfe9b54900 -1 Falling back to public interface 2026-02-20T21:02:59.126 DEBUG:teuthology.orchestra.run.trial199:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-20T21:02:59.389 INFO:teuthology.misc.health.trial199.stderr:2026-02-20T21:02:59.386+0000 7f1f5affd640 0 --2- 10.20.193.199:0/3064718448 >> [v2:10.20.193.199:3300/0,v1:10.20.193.199:6789/0] conn(0x7f1f5c08b100 0x7f1f5c140480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2026-02-20T21:02:59.421 INFO:teuthology.misc.health.trial199.stdout: 2026-02-20T21:02:59.421 INFO:teuthology.misc.health.trial199.stdout:{"epoch":5,"fsid":"f480090c-6109-4036-a7fa-8e7cbf4099eb","created":"2026-02-20T21:02:58.028370+0000","modified":"2026-02-20T21:02:58.696055+0000","last_up_change":"0.000000","last_in_change":"2026-02-20T21:02:58.696055+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"ea3bd762-09f9-4bf7-b187-ff293d2e7ade","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"7b93c7a9-2b3d-4232-9811-774ad9478d1b","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"13b69e07-8c48-41b2-bd6a-7ffe80cec20b","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:02:59.429 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-20T21:02:59.473 INFO:tasks.ceph.cluster2.osd.2.trial199.stderr:2026-02-20T21:02:59.471+0000 7fbfe9b54900 -1 osd.2 0 log_to_monitors true 2026-02-20T21:02:59.519 INFO:tasks.ceph.cluster2.osd.1.trial199.stderr:2026-02-20T21:02:59.517+0000 7fefb8dc9900 -1 osd.1 0 log_to_monitors true 2026-02-20T21:02:59.563 INFO:tasks.ceph.cluster2.osd.0.trial199.stderr:2026-02-20T21:02:59.561+0000 7f16b001c900 -1 osd.0 0 log_to_monitors true 2026-02-20T21:03:00.736 INFO:tasks.ceph.cluster2.mgr.x.trial196.stderr:2026-02-20T21:03:00.734+0000 7f4e5d86c640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-20T21:03:01.045 INFO:tasks.ceph.cluster2.osd.1.trial199.stderr:2026-02-20T21:03:01.043+0000 7fefb3dc3640 -1 osd.1 0 waiting for initial osdmap 2026-02-20T21:03:01.046 INFO:tasks.ceph.cluster2.osd.2.trial199.stderr:2026-02-20T21:03:01.044+0000 7fbfe5ae2640 -1 osd.2 0 waiting for initial osdmap 2026-02-20T21:03:01.047 INFO:tasks.ceph.cluster2.osd.1.trial199.stderr:2026-02-20T21:03:01.045+0000 7fefab3c1640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-20T21:03:01.048 INFO:tasks.ceph.cluster2.osd.2.trial199.stderr:2026-02-20T21:03:01.045+0000 7fbfdd0e0640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-20T21:03:01.050 INFO:tasks.ceph.cluster2.osd.0.trial199.stderr:2026-02-20T21:03:01.048+0000 7f16ab7c3640 -1 osd.0 0 waiting for initial osdmap 2026-02-20T21:03:01.052 INFO:tasks.ceph.cluster2.osd.0.trial199.stderr:2026-02-20T21:03:01.050+0000 7f16a2dc1640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-20T21:03:05.732 DEBUG:teuthology.orchestra.run.trial199:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-20T21:03:05.904 INFO:teuthology.misc.health.trial199.stdout: 2026-02-20T21:03:05.904 INFO:teuthology.misc.health.trial199.stdout:{"epoch":11,"fsid":"f480090c-6109-4036-a7fa-8e7cbf4099eb","created":"2026-02-20T21:02:58.028370+0000","modified":"2026-02-20T21:03:05.041027+0000","last_up_change":"2026-02-20T21:03:02.035109+0000","last_in_change":"2026-02-20T21:02:58.696055+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:03:03.747219+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ea3bd762-09f9-4bf7-b187-ff293d2e7ade","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6808","nonce":2744132852},{"type":"v1","addr":"10.20.193.199:6809","nonce":2744132852}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6810","nonce":2744132852},{"type":"v1","addr":"10.20.193.199:6811","nonce":2744132852}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6816","nonce":2744132852},{"type":"v1","addr":"10.20.193.199:6818","nonce":2744132852}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6812","nonce":2744132852},{"type":"v1","addr":"10.20.193.199:6814","nonce":2744132852}]},"public_addr":"10.20.193.199:6809/2744132852","cluster_addr":"10.20.193.199:6811/2744132852","heartbeat_back_addr":"10.20.193.199:6818/2744132852","heartbeat_front_addr":"10.20.193.199:6814/2744132852","state":["exists","up"]},{"osd":1,"uuid":"7b93c7a9-2b3d-4232-9811-774ad9478d1b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":10,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6800","nonce":2633081233},{"type":"v1","addr":"10.20.193.199:6801","nonce":2633081233}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6802","nonce":2633081233},{"type":"v1","addr":"10.20.193.199:6803","nonce":2633081233}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6806","nonce":2633081233},{"type":"v1","addr":"10.20.193.199:6807","nonce":2633081233}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6804","nonce":2633081233},{"type":"v1","addr":"10.20.193.199:6805","nonce":2633081233}]},"public_addr":"10.20.193.199:6801/2633081233","cluster_addr":"10.20.193.199:6803/2633081233","heartbeat_back_addr":"10.20.193.199:6807/2633081233","heartbeat_front_addr":"10.20.193.199:6805/2633081233","state":["exists","up"]},{"osd":2,"uuid":"13b69e07-8c48-41b2-bd6a-7ffe80cec20b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6813","nonce":3547377386},{"type":"v1","addr":"10.20.193.199:6815","nonce":3547377386}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6817","nonce":3547377386},{"type":"v1","addr":"10.20.193.199:6819","nonce":3547377386}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6822","nonce":3547377386},{"type":"v1","addr":"10.20.193.199:6823","nonce":3547377386}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6820","nonce":3547377386},{"type":"v1","addr":"10.20.193.199:6821","nonce":3547377386}]},"public_addr":"10.20.193.199:6815/3547377386","cluster_addr":"10.20.193.199:6819/3547377386","heartbeat_back_addr":"10.20.193.199:6823/3547377386","heartbeat_front_addr":"10.20.193.199:6821/3547377386","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:03:00.564121+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:03:00.482246+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:03:00.469179+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:03:05.912 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-20T21:03:05.913 INFO:tasks.ceph:Creating RBD pool 2026-02-20T21:03:05.913 DEBUG:teuthology.orchestra.run.trial199:> sudo ceph --cluster cluster2 osd pool create rbd 8 2026-02-20T21:03:07.046 INFO:teuthology.orchestra.run.trial199.stderr:pool 'rbd' created 2026-02-20T21:03:07.060 DEBUG:teuthology.orchestra.run.trial199:> rbd --cluster cluster2 pool init rbd 2026-02-20T21:03:10.060 INFO:tasks.ceph:Starting mds daemons in cluster cluster2... 2026-02-20T21:03:10.060 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 config log 1 --format=json 2026-02-20T21:03:10.061 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-20T21:03:10.242 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:03:10.250 INFO:teuthology.orchestra.run.trial199.stdout:[{"version":1,"timestamp":"0.000000","name":"","changes":[]}] 2026-02-20T21:03:10.251 INFO:tasks.ceph_manager:config epoch is 1 2026-02-20T21:03:10.251 INFO:tasks.ceph:Waiting until cluster2 daemons up and pgs clean... 2026-02-20T21:03:10.251 INFO:tasks.ceph.ceph_manager.cluster2:waiting for mgr available 2026-02-20T21:03:10.251 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 mgr dump --format=json 2026-02-20T21:03:10.463 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:03:10.473 INFO:teuthology.orchestra.run.trial199.stdout:{"epoch":5,"flags":0,"active_gid":4102,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6826","nonce":694943720},{"type":"v1","addr":"10.20.193.196:6827","nonce":694943720}]},"active_addr":"10.20.193.196:6827/694943720","active_change":"2026-02-20T21:02:59.728561+0000","active_mgr_features":4544132024016699391,"available":true,"standbys":[],"modules":["iostat","nfs"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"certificate_automated_rotation_enabled":{"name":"certificate_automated_rotation_enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"This flag controls whether cephadm automatically rotates certificates upon expiration.","long_desc":"","tags":[],"see_also":[]},"certificate_check_debug_mode":{"name":"certificate_check_debug_mode","type":"bool","level":"dev","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"FOR TESTING ONLY: This flag forces the certificate check instead of waiting for certificate_check_period.","long_desc":"","tags":[],"see_also":[]},"certificate_check_period":{"name":"certificate_check_period","type":"int","level":"advanced","flags":0,"default_value":"1","min":"0","max":"30","enum_allowed":[],"desc":"Specifies how often (in days) the certificate should be checked for validity.","long_desc":"","tags":[],"see_also":[]},"certificate_duration_days":{"name":"certificate_duration_days","type":"int","level":"advanced","flags":0,"default_value":"1095","min":"90","max":"3650","enum_allowed":[],"desc":"Specifies the duration of self certificates generated and signed by cephadm root CA","long_desc":"","tags":[],"see_also":[]},"certificate_renewal_threshold_days":{"name":"certificate_renewal_threshold_days","type":"int","level":"advanced","flags":0,"default_value":"30","min":"10","max":"90","enum_allowed":[],"desc":"Specifies the lead time in days to initiate certificate renewal before expiration.","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.28.1","min":"","max":"","enum_allowed":[],"desc":"Alertmanager container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"Elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:12.3.1","min":"","max":"","enum_allowed":[],"desc":"Grafana container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"Haproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.9.1","min":"","max":"","enum_allowed":[],"desc":"Node exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.5","min":"","max":"","enum_allowed":[],"desc":"Nvmeof container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"Oauth2 proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v3.6.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba metrics container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"Snmp gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"stray_daemon_check_interval":{"name":"stray_daemon_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"how frequently cephadm should check for the presence of stray daemons","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROM_ALERT_CREDENTIAL_CACHE_TTL":{"name":"PROM_ALERT_CREDENTIAL_CACHE_TTL","type":"int","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_HOSTNAME_PER_DAEMON":{"name":"RGW_HOSTNAME_PER_DAEMON","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crypto_caller":{"name":"crypto_caller","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"prometheus_tls_secret_name":{"name":"prometheus_tls_secret_name","type":"str","level":"advanced","flags":0,"default_value":"rook-ceph-prometheus-server-tls","min":"","max":"","enum_allowed":[],"desc":"name of tls secret in k8s for prometheus","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_cloning":{"name":"pause_cloning","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_purging":{"name":"pause_purging","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous subvolume purge threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"tentacle":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.196:0","nonce":962969554}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.196:0","nonce":2760427859}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.196:0","nonce":2331835762}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.196:0","nonce":1214505601}]}]} 2026-02-20T21:03:10.480 INFO:tasks.ceph.ceph_manager.cluster2:mgr available! 2026-02-20T21:03:10.480 INFO:tasks.ceph.ceph_manager.cluster2:waiting for all up 2026-02-20T21:03:10.480 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-20T21:03:10.661 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:03:10.662 INFO:teuthology.orchestra.run.trial199.stdout:{"epoch":16,"fsid":"f480090c-6109-4036-a7fa-8e7cbf4099eb","created":"2026-02-20T21:02:58.028370+0000","modified":"2026-02-20T21:03:10.049963+0000","last_up_change":"2026-02-20T21:03:02.035109+0000","last_in_change":"2026-02-20T21:02:58.696055+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:03:03.747219+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-20T21:03:06.093359+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ea3bd762-09f9-4bf7-b187-ff293d2e7ade","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6808","nonce":2744132852},{"type":"v1","addr":"10.20.193.199:6809","nonce":2744132852}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6810","nonce":2744132852},{"type":"v1","addr":"10.20.193.199:6811","nonce":2744132852}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6816","nonce":2744132852},{"type":"v1","addr":"10.20.193.199:6818","nonce":2744132852}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6812","nonce":2744132852},{"type":"v1","addr":"10.20.193.199:6814","nonce":2744132852}]},"public_addr":"10.20.193.199:6809/2744132852","cluster_addr":"10.20.193.199:6811/2744132852","heartbeat_back_addr":"10.20.193.199:6818/2744132852","heartbeat_front_addr":"10.20.193.199:6814/2744132852","state":["exists","up"]},{"osd":1,"uuid":"7b93c7a9-2b3d-4232-9811-774ad9478d1b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6800","nonce":2633081233},{"type":"v1","addr":"10.20.193.199:6801","nonce":2633081233}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6802","nonce":2633081233},{"type":"v1","addr":"10.20.193.199:6803","nonce":2633081233}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6806","nonce":2633081233},{"type":"v1","addr":"10.20.193.199:6807","nonce":2633081233}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6804","nonce":2633081233},{"type":"v1","addr":"10.20.193.199:6805","nonce":2633081233}]},"public_addr":"10.20.193.199:6801/2633081233","cluster_addr":"10.20.193.199:6803/2633081233","heartbeat_back_addr":"10.20.193.199:6807/2633081233","heartbeat_front_addr":"10.20.193.199:6805/2633081233","state":["exists","up"]},{"osd":2,"uuid":"13b69e07-8c48-41b2-bd6a-7ffe80cec20b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6813","nonce":3547377386},{"type":"v1","addr":"10.20.193.199:6815","nonce":3547377386}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6817","nonce":3547377386},{"type":"v1","addr":"10.20.193.199:6819","nonce":3547377386}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6822","nonce":3547377386},{"type":"v1","addr":"10.20.193.199:6823","nonce":3547377386}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6820","nonce":3547377386},{"type":"v1","addr":"10.20.193.199:6821","nonce":3547377386}]},"public_addr":"10.20.193.199:6815/3547377386","cluster_addr":"10.20.193.199:6819/3547377386","heartbeat_back_addr":"10.20.193.199:6823/3547377386","heartbeat_front_addr":"10.20.193.199:6821/3547377386","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:03:00.564121+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:03:00.482246+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:03:00.469179+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:03:10.671 INFO:tasks.ceph.ceph_manager.cluster2:all up! 2026-02-20T21:03:10.671 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-20T21:03:10.853 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:03:10.854 INFO:teuthology.orchestra.run.trial199.stdout:{"epoch":16,"fsid":"f480090c-6109-4036-a7fa-8e7cbf4099eb","created":"2026-02-20T21:02:58.028370+0000","modified":"2026-02-20T21:03:10.049963+0000","last_up_change":"2026-02-20T21:03:02.035109+0000","last_in_change":"2026-02-20T21:02:58.696055+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:03:03.747219+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-20T21:03:06.093359+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ea3bd762-09f9-4bf7-b187-ff293d2e7ade","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6808","nonce":2744132852},{"type":"v1","addr":"10.20.193.199:6809","nonce":2744132852}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6810","nonce":2744132852},{"type":"v1","addr":"10.20.193.199:6811","nonce":2744132852}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6816","nonce":2744132852},{"type":"v1","addr":"10.20.193.199:6818","nonce":2744132852}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6812","nonce":2744132852},{"type":"v1","addr":"10.20.193.199:6814","nonce":2744132852}]},"public_addr":"10.20.193.199:6809/2744132852","cluster_addr":"10.20.193.199:6811/2744132852","heartbeat_back_addr":"10.20.193.199:6818/2744132852","heartbeat_front_addr":"10.20.193.199:6814/2744132852","state":["exists","up"]},{"osd":1,"uuid":"7b93c7a9-2b3d-4232-9811-774ad9478d1b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6800","nonce":2633081233},{"type":"v1","addr":"10.20.193.199:6801","nonce":2633081233}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6802","nonce":2633081233},{"type":"v1","addr":"10.20.193.199:6803","nonce":2633081233}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6806","nonce":2633081233},{"type":"v1","addr":"10.20.193.199:6807","nonce":2633081233}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6804","nonce":2633081233},{"type":"v1","addr":"10.20.193.199:6805","nonce":2633081233}]},"public_addr":"10.20.193.199:6801/2633081233","cluster_addr":"10.20.193.199:6803/2633081233","heartbeat_back_addr":"10.20.193.199:6807/2633081233","heartbeat_front_addr":"10.20.193.199:6805/2633081233","state":["exists","up"]},{"osd":2,"uuid":"13b69e07-8c48-41b2-bd6a-7ffe80cec20b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6813","nonce":3547377386},{"type":"v1","addr":"10.20.193.199:6815","nonce":3547377386}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6817","nonce":3547377386},{"type":"v1","addr":"10.20.193.199:6819","nonce":3547377386}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6822","nonce":3547377386},{"type":"v1","addr":"10.20.193.199:6823","nonce":3547377386}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6820","nonce":3547377386},{"type":"v1","addr":"10.20.193.199:6821","nonce":3547377386}]},"public_addr":"10.20.193.199:6815/3547377386","cluster_addr":"10.20.193.199:6819/3547377386","heartbeat_back_addr":"10.20.193.199:6823/3547377386","heartbeat_front_addr":"10.20.193.199:6821/3547377386","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:03:00.564121+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:03:00.482246+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:03:00.469179+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:03:10.864 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 flush_pg_stats 2026-02-20T21:03:10.865 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 flush_pg_stats 2026-02-20T21:03:10.865 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 flush_pg_stats 2026-02-20T21:03:10.958 INFO:teuthology.orchestra.run.trial199.stdout:34359738372 2026-02-20T21:03:10.958 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-20T21:03:10.960 INFO:teuthology.orchestra.run.trial199.stdout:34359738372 2026-02-20T21:03:10.960 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-20T21:03:10.961 INFO:teuthology.orchestra.run.trial199.stdout:34359738373 2026-02-20T21:03:10.962 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-20T21:03:11.185 INFO:teuthology.orchestra.run.trial199.stdout:34359738371 2026-02-20T21:03:11.193 INFO:teuthology.orchestra.run.trial199.stdout:34359738371 2026-02-20T21:03:11.193 INFO:teuthology.orchestra.run.trial199.stdout:34359738372 2026-02-20T21:03:11.195 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.2 2026-02-20T21:03:11.203 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738372 for osd.1 2026-02-20T21:03:11.204 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.0 2026-02-20T21:03:12.196 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-20T21:03:12.204 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-20T21:03:12.204 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-20T21:03:12.384 INFO:teuthology.orchestra.run.trial199.stdout:34359738372 2026-02-20T21:03:12.395 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738372 for osd.2 2026-02-20T21:03:12.396 DEBUG:teuthology.parallel:result is None 2026-02-20T21:03:12.425 INFO:teuthology.orchestra.run.trial199.stdout:34359738372 2026-02-20T21:03:12.430 INFO:teuthology.orchestra.run.trial199.stdout:34359738373 2026-02-20T21:03:12.434 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738372 for osd.0 2026-02-20T21:03:12.435 DEBUG:teuthology.parallel:result is None 2026-02-20T21:03:12.440 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738373 for osd.1 2026-02-20T21:03:12.440 DEBUG:teuthology.parallel:result is None 2026-02-20T21:03:12.441 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-20T21:03:12.441 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-20T21:03:12.660 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:03:12.661 INFO:teuthology.orchestra.run.trial199.stderr:dumped all 2026-02-20T21:03:12.669 INFO:teuthology.orchestra.run.trial199.stdout:{"pg_ready":true,"pg_map":{"version":19,"stamp":"2026-02-20T21:03:11.733668+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":13,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81416,"kb_used_data":536,"kb_used_omap":25,"kb_used_meta":80806,"kb_avail":283034104,"statfs":{"total":289910292480,"available":289826922496,"internally_reserved":0,"allocated":548864,"data_stored":1299439,"data_compressed":67134,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":26325,"internal_metadata":82745643},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.684814"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:03:10.052751+0000","last_change":"2026-02-20T21:03:10.052984+0000","last_active":"2026-02-20T21:03:10.052751+0000","last_peered":"2026-02-20T21:03:10.052751+0000","last_clean":"2026-02-20T21:03:10.052751+0000","last_became_active":"2026-02-20T21:03:08.051994+0000","last_became_peered":"2026-02-20T21:03:08.051994+0000","last_unstale":"2026-02-20T21:03:10.052751+0000","last_undegraded":"2026-02-20T21:03:10.052751+0000","last_fullsized":"2026-02-20T21:03:10.052751+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:23:12.739248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061609399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:03:10.052912+0000","last_change":"2026-02-20T21:03:10.053169+0000","last_active":"2026-02-20T21:03:10.052912+0000","last_peered":"2026-02-20T21:03:10.052912+0000","last_clean":"2026-02-20T21:03:10.052912+0000","last_became_active":"2026-02-20T21:03:08.052053+0000","last_became_peered":"2026-02-20T21:03:08.052053+0000","last_unstale":"2026-02-20T21:03:10.052912+0000","last_undegraded":"2026-02-20T21:03:10.052912+0000","last_fullsized":"2026-02-20T21:03:10.052912+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:22:33.108819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079631999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:03:10.052675+0000","last_change":"2026-02-20T21:03:10.052860+0000","last_active":"2026-02-20T21:03:10.052675+0000","last_peered":"2026-02-20T21:03:10.052675+0000","last_clean":"2026-02-20T21:03:10.052675+0000","last_became_active":"2026-02-20T21:03:08.051990+0000","last_became_peered":"2026-02-20T21:03:08.051990+0000","last_unstale":"2026-02-20T21:03:10.052675+0000","last_undegraded":"2026-02-20T21:03:10.052675+0000","last_fullsized":"2026-02-20T21:03:10.052675+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:56:43.086257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050303499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:03:10.052484+0000","last_change":"2026-02-20T21:03:10.052617+0000","last_active":"2026-02-20T21:03:10.052484+0000","last_peered":"2026-02-20T21:03:10.052484+0000","last_clean":"2026-02-20T21:03:10.052484+0000","last_became_active":"2026-02-20T21:03:08.052009+0000","last_became_peered":"2026-02-20T21:03:08.052009+0000","last_unstale":"2026-02-20T21:03:10.052484+0000","last_undegraded":"2026-02-20T21:03:10.052484+0000","last_fullsized":"2026-02-20T21:03:10.052484+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:48:37.573173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039422300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":22,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:03:10.054831+0000","last_change":"2026-02-20T21:03:10.054541+0000","last_active":"2026-02-20T21:03:10.054831+0000","last_peered":"2026-02-20T21:03:10.054831+0000","last_clean":"2026-02-20T21:03:10.054831+0000","last_became_active":"2026-02-20T21:03:08.052246+0000","last_became_peered":"2026-02-20T21:03:08.052246+0000","last_unstale":"2026-02-20T21:03:10.054831+0000","last_undegraded":"2026-02-20T21:03:10.054831+0000","last_fullsized":"2026-02-20T21:03:10.054831+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:02:15.970404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00076615400000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:03:10.436905+0000","last_change":"2026-02-20T21:03:10.437050+0000","last_active":"2026-02-20T21:03:10.436905+0000","last_peered":"2026-02-20T21:03:10.436905+0000","last_clean":"2026-02-20T21:03:10.436905+0000","last_became_active":"2026-02-20T21:03:08.052284+0000","last_became_peered":"2026-02-20T21:03:08.052284+0000","last_unstale":"2026-02-20T21:03:10.436905+0000","last_undegraded":"2026-02-20T21:03:10.436905+0000","last_fullsized":"2026-02-20T21:03:10.436905+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:53:33.417610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030119000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:03:10.436871+0000","last_change":"2026-02-20T21:03:10.436978+0000","last_active":"2026-02-20T21:03:10.436871+0000","last_peered":"2026-02-20T21:03:10.436871+0000","last_clean":"2026-02-20T21:03:10.436871+0000","last_became_active":"2026-02-20T21:03:08.052106+0000","last_became_peered":"2026-02-20T21:03:08.052106+0000","last_unstale":"2026-02-20T21:03:10.436871+0000","last_undegraded":"2026-02-20T21:03:10.436871+0000","last_fullsized":"2026-02-20T21:03:10.436871+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:47:18.795582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000257268,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:03:10.052538+0000","last_change":"2026-02-20T21:03:10.052664+0000","last_active":"2026-02-20T21:03:10.052538+0000","last_peered":"2026-02-20T21:03:10.052538+0000","last_clean":"2026-02-20T21:03:10.052538+0000","last_became_active":"2026-02-20T21:03:08.051931+0000","last_became_peered":"2026-02-20T21:03:08.051931+0000","last_unstale":"2026-02-20T21:03:10.052538+0000","last_undegraded":"2026-02-20T21:03:10.052538+0000","last_fullsized":"2026-02-20T21:03:10.052538+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:39:20.931975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000310737,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'72","reported_seq":111,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:03:10.052159+0000","last_change":"2026-02-20T21:03:05.421656+0000","last_active":"2026-02-20T21:03:10.052159+0000","last_peered":"2026-02-20T21:03:10.052159+0000","last_clean":"2026-02-20T21:03:10.052159+0000","last_became_active":"2026-02-20T21:03:05.421208+0000","last_became_peered":"2026-02-20T21:03:05.421208+0000","last_unstale":"2026-02-20T21:03:10.052159+0000","last_undegraded":"2026-02-20T21:03:10.052159+0000","last_fullsized":"2026-02-20T21:03:10.052159+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:04.040085+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:04.040085+0000","last_clean_scrub_stamp":"2026-02-20T21:03:04.040085+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:35:31.987173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67134,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738372,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27088,"kb_used_data":128,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344752,"statfs":{"total":96636764160,"available":96609026048,"internally_reserved":0,"allocated":131072,"data_stored":37179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738373,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27164,"kb_used_data":204,"kb_used_omap":9,"kb_used_meta":26934,"kb_avail":94344676,"statfs":{"total":96636764160,"available":96608948224,"internally_reserved":0,"allocated":208896,"data_stored":631130,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":9425,"internal_metadata":27581231},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27164,"kb_used_data":204,"kb_used_omap":8,"kb_used_meta":26935,"kb_avail":94344676,"statfs":{"total":96636764160,"available":96608948224,"internally_reserved":0,"allocated":208896,"data_stored":631130,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":8775,"internal_metadata":27581881},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:03:12.671 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-20T21:03:12.851 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:03:12.851 INFO:teuthology.orchestra.run.trial199.stderr:dumped all 2026-02-20T21:03:12.859 INFO:teuthology.orchestra.run.trial199.stdout:{"pg_ready":true,"pg_map":{"version":19,"stamp":"2026-02-20T21:03:11.733668+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":13,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81416,"kb_used_data":536,"kb_used_omap":25,"kb_used_meta":80806,"kb_avail":283034104,"statfs":{"total":289910292480,"available":289826922496,"internally_reserved":0,"allocated":548864,"data_stored":1299439,"data_compressed":67134,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":26325,"internal_metadata":82745643},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.684814"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:03:10.052751+0000","last_change":"2026-02-20T21:03:10.052984+0000","last_active":"2026-02-20T21:03:10.052751+0000","last_peered":"2026-02-20T21:03:10.052751+0000","last_clean":"2026-02-20T21:03:10.052751+0000","last_became_active":"2026-02-20T21:03:08.051994+0000","last_became_peered":"2026-02-20T21:03:08.051994+0000","last_unstale":"2026-02-20T21:03:10.052751+0000","last_undegraded":"2026-02-20T21:03:10.052751+0000","last_fullsized":"2026-02-20T21:03:10.052751+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:23:12.739248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061609399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:03:10.052912+0000","last_change":"2026-02-20T21:03:10.053169+0000","last_active":"2026-02-20T21:03:10.052912+0000","last_peered":"2026-02-20T21:03:10.052912+0000","last_clean":"2026-02-20T21:03:10.052912+0000","last_became_active":"2026-02-20T21:03:08.052053+0000","last_became_peered":"2026-02-20T21:03:08.052053+0000","last_unstale":"2026-02-20T21:03:10.052912+0000","last_undegraded":"2026-02-20T21:03:10.052912+0000","last_fullsized":"2026-02-20T21:03:10.052912+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:22:33.108819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079631999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:03:10.052675+0000","last_change":"2026-02-20T21:03:10.052860+0000","last_active":"2026-02-20T21:03:10.052675+0000","last_peered":"2026-02-20T21:03:10.052675+0000","last_clean":"2026-02-20T21:03:10.052675+0000","last_became_active":"2026-02-20T21:03:08.051990+0000","last_became_peered":"2026-02-20T21:03:08.051990+0000","last_unstale":"2026-02-20T21:03:10.052675+0000","last_undegraded":"2026-02-20T21:03:10.052675+0000","last_fullsized":"2026-02-20T21:03:10.052675+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:56:43.086257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050303499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:03:10.052484+0000","last_change":"2026-02-20T21:03:10.052617+0000","last_active":"2026-02-20T21:03:10.052484+0000","last_peered":"2026-02-20T21:03:10.052484+0000","last_clean":"2026-02-20T21:03:10.052484+0000","last_became_active":"2026-02-20T21:03:08.052009+0000","last_became_peered":"2026-02-20T21:03:08.052009+0000","last_unstale":"2026-02-20T21:03:10.052484+0000","last_undegraded":"2026-02-20T21:03:10.052484+0000","last_fullsized":"2026-02-20T21:03:10.052484+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:48:37.573173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039422300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":22,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:03:10.054831+0000","last_change":"2026-02-20T21:03:10.054541+0000","last_active":"2026-02-20T21:03:10.054831+0000","last_peered":"2026-02-20T21:03:10.054831+0000","last_clean":"2026-02-20T21:03:10.054831+0000","last_became_active":"2026-02-20T21:03:08.052246+0000","last_became_peered":"2026-02-20T21:03:08.052246+0000","last_unstale":"2026-02-20T21:03:10.054831+0000","last_undegraded":"2026-02-20T21:03:10.054831+0000","last_fullsized":"2026-02-20T21:03:10.054831+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:02:15.970404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00076615400000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:03:10.436905+0000","last_change":"2026-02-20T21:03:10.437050+0000","last_active":"2026-02-20T21:03:10.436905+0000","last_peered":"2026-02-20T21:03:10.436905+0000","last_clean":"2026-02-20T21:03:10.436905+0000","last_became_active":"2026-02-20T21:03:08.052284+0000","last_became_peered":"2026-02-20T21:03:08.052284+0000","last_unstale":"2026-02-20T21:03:10.436905+0000","last_undegraded":"2026-02-20T21:03:10.436905+0000","last_fullsized":"2026-02-20T21:03:10.436905+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:53:33.417610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030119000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:03:10.436871+0000","last_change":"2026-02-20T21:03:10.436978+0000","last_active":"2026-02-20T21:03:10.436871+0000","last_peered":"2026-02-20T21:03:10.436871+0000","last_clean":"2026-02-20T21:03:10.436871+0000","last_became_active":"2026-02-20T21:03:08.052106+0000","last_became_peered":"2026-02-20T21:03:08.052106+0000","last_unstale":"2026-02-20T21:03:10.436871+0000","last_undegraded":"2026-02-20T21:03:10.436871+0000","last_fullsized":"2026-02-20T21:03:10.436871+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:47:18.795582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000257268,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:03:10.052538+0000","last_change":"2026-02-20T21:03:10.052664+0000","last_active":"2026-02-20T21:03:10.052538+0000","last_peered":"2026-02-20T21:03:10.052538+0000","last_clean":"2026-02-20T21:03:10.052538+0000","last_became_active":"2026-02-20T21:03:08.051931+0000","last_became_peered":"2026-02-20T21:03:08.051931+0000","last_unstale":"2026-02-20T21:03:10.052538+0000","last_undegraded":"2026-02-20T21:03:10.052538+0000","last_fullsized":"2026-02-20T21:03:10.052538+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:39:20.931975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000310737,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'72","reported_seq":111,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:03:10.052159+0000","last_change":"2026-02-20T21:03:05.421656+0000","last_active":"2026-02-20T21:03:10.052159+0000","last_peered":"2026-02-20T21:03:10.052159+0000","last_clean":"2026-02-20T21:03:10.052159+0000","last_became_active":"2026-02-20T21:03:05.421208+0000","last_became_peered":"2026-02-20T21:03:05.421208+0000","last_unstale":"2026-02-20T21:03:10.052159+0000","last_undegraded":"2026-02-20T21:03:10.052159+0000","last_fullsized":"2026-02-20T21:03:10.052159+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:04.040085+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:04.040085+0000","last_clean_scrub_stamp":"2026-02-20T21:03:04.040085+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:35:31.987173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67134,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738372,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27088,"kb_used_data":128,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344752,"statfs":{"total":96636764160,"available":96609026048,"internally_reserved":0,"allocated":131072,"data_stored":37179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738373,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27164,"kb_used_data":204,"kb_used_omap":9,"kb_used_meta":26934,"kb_avail":94344676,"statfs":{"total":96636764160,"available":96608948224,"internally_reserved":0,"allocated":208896,"data_stored":631130,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":9425,"internal_metadata":27581231},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27164,"kb_used_data":204,"kb_used_omap":8,"kb_used_meta":26935,"kb_avail":94344676,"statfs":{"total":96636764160,"available":96608948224,"internally_reserved":0,"allocated":208896,"data_stored":631130,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":8775,"internal_metadata":27581881},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:03:12.861 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-20T21:03:12.861 INFO:tasks.ceph:Waiting until ceph cluster cluster2 is healthy... 2026-02-20T21:03:12.861 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy 2026-02-20T21:03:12.861 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 health --format=json 2026-02-20T21:03:13.065 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:03:13.065 INFO:teuthology.orchestra.run.trial199.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-20T21:03:13.075 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy done 2026-02-20T21:03:13.075 INFO:teuthology.run_tasks:Running task exec... 2026-02-20T21:03:13.088 INFO:teuthology.task.exec:Executing custom commands... 2026-02-20T21:03:13.088 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@trial199.front.sepia.ceph.com 2026-02-20T21:03:13.088 DEBUG:teuthology.orchestra.run.trial199:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.4 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:03:13.317 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.4] 2026-02-20T21:03:13.318 INFO:teuthology.orchestra.run.trial199.stdout: key = AQDiy5hpMOlwCxAA2svqHAJIU4dXKiKu9Uqxpg== 2026-02-20T21:03:13.318 INFO:teuthology.orchestra.run.trial199.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:03:13.318 INFO:teuthology.orchestra.run.trial199.stdout: caps osd = "profile rbd" 2026-02-20T21:03:13.318 INFO:teuthology.orchestra.run.trial199.stderr:updated caps for client.mirror.4 2026-02-20T21:03:13.327 DEBUG:teuthology.orchestra.run.trial199:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.5 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:03:13.554 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.5] 2026-02-20T21:03:13.554 INFO:teuthology.orchestra.run.trial199.stdout: key = AQDiy5hpxq6WEBAAe9nMMfjqDxsJRFou19tZsA== 2026-02-20T21:03:13.554 INFO:teuthology.orchestra.run.trial199.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:03:13.555 INFO:teuthology.orchestra.run.trial199.stdout: caps osd = "profile rbd" 2026-02-20T21:03:13.555 INFO:teuthology.orchestra.run.trial199.stderr:updated caps for client.mirror.5 2026-02-20T21:03:13.565 DEBUG:teuthology.orchestra.run.trial199:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.6 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:03:13.794 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.6] 2026-02-20T21:03:13.794 INFO:teuthology.orchestra.run.trial199.stdout: key = AQDiy5hpYjC8FRAAMjXbeu0X7XY0mBnAFiuphA== 2026-02-20T21:03:13.794 INFO:teuthology.orchestra.run.trial199.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:03:13.794 INFO:teuthology.orchestra.run.trial199.stdout: caps osd = "profile rbd" 2026-02-20T21:03:13.795 INFO:teuthology.orchestra.run.trial199.stderr:updated caps for client.mirror.6 2026-02-20T21:03:13.804 DEBUG:teuthology.orchestra.run.trial199:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.4 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:03:14.034 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.4] 2026-02-20T21:03:14.035 INFO:teuthology.orchestra.run.trial199.stdout: key = AQD7y5hpLIFEFxAA1+wNdmlvRMvGOz0bRm7+ug== 2026-02-20T21:03:14.035 INFO:teuthology.orchestra.run.trial199.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:03:14.035 INFO:teuthology.orchestra.run.trial199.stdout: caps osd = "profile rbd" 2026-02-20T21:03:14.035 INFO:teuthology.orchestra.run.trial199.stderr:updated caps for client.mirror.4 2026-02-20T21:03:14.045 DEBUG:teuthology.orchestra.run.trial199:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.5 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:03:14.274 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.5] 2026-02-20T21:03:14.275 INFO:teuthology.orchestra.run.trial199.stdout: key = AQD7y5hpISFWHBAAHrUnn8ssEvp6oL69w8Nd8g== 2026-02-20T21:03:14.275 INFO:teuthology.orchestra.run.trial199.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:03:14.275 INFO:teuthology.orchestra.run.trial199.stdout: caps osd = "profile rbd" 2026-02-20T21:03:14.275 INFO:teuthology.orchestra.run.trial199.stderr:updated caps for client.mirror.5 2026-02-20T21:03:14.285 DEBUG:teuthology.orchestra.run.trial199:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.6 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:03:14.513 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.6] 2026-02-20T21:03:14.513 INFO:teuthology.orchestra.run.trial199.stdout: key = AQD7y5hp7rJYIRAAL0Q5kIzSqnIGA6lgpx28HA== 2026-02-20T21:03:14.513 INFO:teuthology.orchestra.run.trial199.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:03:14.513 INFO:teuthology.orchestra.run.trial199.stdout: caps osd = "profile rbd" 2026-02-20T21:03:14.513 INFO:teuthology.orchestra.run.trial199.stderr:updated caps for client.mirror.6 2026-02-20T21:03:14.524 INFO:teuthology.run_tasks:Running task exec... 2026-02-20T21:03:14.538 INFO:teuthology.task.exec:Executing custom commands... 2026-02-20T21:03:14.538 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@trial199.front.sepia.ceph.com 2026-02-20T21:03:14.538 DEBUG:teuthology.orchestra.run.trial199:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:03:14.764 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror] 2026-02-20T21:03:14.765 INFO:teuthology.orchestra.run.trial199.stdout: key = AQDhy5hpzZRFLRAAZCep4c18uP5QCY+heo+0PA== 2026-02-20T21:03:14.765 INFO:teuthology.orchestra.run.trial199.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-20T21:03:14.765 INFO:teuthology.orchestra.run.trial199.stdout: caps osd = "profile rbd" 2026-02-20T21:03:14.765 INFO:teuthology.orchestra.run.trial199.stderr:updated caps for client.mirror 2026-02-20T21:03:14.774 DEBUG:teuthology.orchestra.run.trial199:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:03:15.008 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.0] 2026-02-20T21:03:15.008 INFO:teuthology.orchestra.run.trial199.stdout: key = AQDhy5hpGBptMhAAXIxDxKmuIWVPT0qg2J8Wfg== 2026-02-20T21:03:15.008 INFO:teuthology.orchestra.run.trial199.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:03:15.008 INFO:teuthology.orchestra.run.trial199.stdout: caps osd = "profile rbd" 2026-02-20T21:03:15.008 INFO:teuthology.orchestra.run.trial199.stderr:updated caps for client.mirror.0 2026-02-20T21:03:15.018 DEBUG:teuthology.orchestra.run.trial199:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:03:15.247 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.1] 2026-02-20T21:03:15.247 INFO:teuthology.orchestra.run.trial199.stdout: key = AQDhy5hpc4yWNxAAaR5N8NvCR6cqM2D7xTaoCg== 2026-02-20T21:03:15.247 INFO:teuthology.orchestra.run.trial199.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:03:15.247 INFO:teuthology.orchestra.run.trial199.stdout: caps osd = "profile rbd" 2026-02-20T21:03:15.247 INFO:teuthology.orchestra.run.trial199.stderr:updated caps for client.mirror.1 2026-02-20T21:03:15.257 DEBUG:teuthology.orchestra.run.trial199:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:03:15.488 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.2] 2026-02-20T21:03:15.488 INFO:teuthology.orchestra.run.trial199.stdout: key = AQDiy5hpMwAOARAAXeusCfPWgDz0u9Kc6qvs+Q== 2026-02-20T21:03:15.488 INFO:teuthology.orchestra.run.trial199.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:03:15.488 INFO:teuthology.orchestra.run.trial199.stdout: caps osd = "profile rbd" 2026-02-20T21:03:15.488 INFO:teuthology.orchestra.run.trial199.stderr:updated caps for client.mirror.2 2026-02-20T21:03:15.498 DEBUG:teuthology.orchestra.run.trial199:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:03:15.725 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.3] 2026-02-20T21:03:15.725 INFO:teuthology.orchestra.run.trial199.stdout: key = AQDiy5hpG/xNBhAAf8aXhKl8gDTMi+kWbkzfMA== 2026-02-20T21:03:15.725 INFO:teuthology.orchestra.run.trial199.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:03:15.725 INFO:teuthology.orchestra.run.trial199.stdout: caps osd = "profile rbd" 2026-02-20T21:03:15.726 INFO:teuthology.orchestra.run.trial199.stderr:updated caps for client.mirror.3 2026-02-20T21:03:15.735 DEBUG:teuthology.orchestra.run.trial199:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:03:15.965 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror] 2026-02-20T21:03:15.965 INFO:teuthology.orchestra.run.trial199.stdout: key = AQD6y5hpbKErORAAxF/VX2dLPA47r7ykjtzV2Q== 2026-02-20T21:03:15.965 INFO:teuthology.orchestra.run.trial199.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-20T21:03:15.965 INFO:teuthology.orchestra.run.trial199.stdout: caps osd = "profile rbd" 2026-02-20T21:03:15.965 INFO:teuthology.orchestra.run.trial199.stderr:updated caps for client.mirror 2026-02-20T21:03:15.976 DEBUG:teuthology.orchestra.run.trial199:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:03:16.201 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.0] 2026-02-20T21:03:16.202 INFO:teuthology.orchestra.run.trial199.stdout: key = AQD7y5hpz2aqAhAARIW7sEk5C4HBhYBtw4xe2Q== 2026-02-20T21:03:16.202 INFO:teuthology.orchestra.run.trial199.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:03:16.202 INFO:teuthology.orchestra.run.trial199.stdout: caps osd = "profile rbd" 2026-02-20T21:03:16.202 INFO:teuthology.orchestra.run.trial199.stderr:updated caps for client.mirror.0 2026-02-20T21:03:16.212 DEBUG:teuthology.orchestra.run.trial199:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:03:16.443 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.1] 2026-02-20T21:03:16.443 INFO:teuthology.orchestra.run.trial199.stdout: key = AQD7y5hpiF7sBxAAQ8ndp7vJ7QJN9ZsxRdB1IQ== 2026-02-20T21:03:16.444 INFO:teuthology.orchestra.run.trial199.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:03:16.444 INFO:teuthology.orchestra.run.trial199.stdout: caps osd = "profile rbd" 2026-02-20T21:03:16.444 INFO:teuthology.orchestra.run.trial199.stderr:updated caps for client.mirror.1 2026-02-20T21:03:16.454 DEBUG:teuthology.orchestra.run.trial199:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:03:16.687 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.2] 2026-02-20T21:03:16.687 INFO:teuthology.orchestra.run.trial199.stdout: key = AQD7y5hpjAkZDRAAIWy2xbLePOOnOK1o/SrTWw== 2026-02-20T21:03:16.687 INFO:teuthology.orchestra.run.trial199.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:03:16.687 INFO:teuthology.orchestra.run.trial199.stdout: caps osd = "profile rbd" 2026-02-20T21:03:16.687 INFO:teuthology.orchestra.run.trial199.stderr:updated caps for client.mirror.2 2026-02-20T21:03:16.698 DEBUG:teuthology.orchestra.run.trial199:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:03:16.928 INFO:teuthology.orchestra.run.trial199.stdout:[client.mirror.3] 2026-02-20T21:03:16.928 INFO:teuthology.orchestra.run.trial199.stdout: key = AQD7y5hpZVUvEhAA7EBX09mGGZujE/FsO2e8XA== 2026-02-20T21:03:16.928 INFO:teuthology.orchestra.run.trial199.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:03:16.928 INFO:teuthology.orchestra.run.trial199.stdout: caps osd = "profile rbd" 2026-02-20T21:03:16.928 INFO:teuthology.orchestra.run.trial199.stderr:updated caps for client.mirror.3 2026-02-20T21:03:16.938 INFO:teuthology.run_tasks:Running task workunit... 2026-02-20T21:03:16.954 INFO:tasks.workunit:Pulling workunits from ref f7870454149f22574086dd2ff55d050585136531 2026-02-20T21:03:16.954 INFO:tasks.workunit:Making a separate scratch dir for every client... 2026-02-20T21:03:16.954 DEBUG:teuthology.orchestra.run.trial199:> stat -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-20T21:03:16.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T21:03:16.971 INFO:teuthology.orchestra.run.trial199.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.cluster1.mirror': No such file or directory 2026-02-20T21:03:16.972 DEBUG:teuthology.orchestra.run.trial199:> mkdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-20T21:03:17.027 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-20T21:03:17.028 DEBUG:teuthology.orchestra.run.trial199:> cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror && mkdir -- client.mirror 2026-02-20T21:03:17.084 INFO:tasks.workunit:timeout=3h 2026-02-20T21:03:17.084 INFO:tasks.workunit:cleanup=True 2026-02-20T21:03:17.084 DEBUG:teuthology.orchestra.run.trial199:> rm -rf /home/ubuntu/cephtest/clone.cluster1.client.mirror && git clone https://github.com/ceph/ceph-ci.git /home/ubuntu/cephtest/clone.cluster1.client.mirror && cd /home/ubuntu/cephtest/clone.cluster1.client.mirror && git checkout f7870454149f22574086dd2ff55d050585136531 2026-02-20T21:03:17.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Cloning into '/home/ubuntu/cephtest/clone.cluster1.client.mirror'... 2026-02-20T21:03:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Note: switching to 'f7870454149f22574086dd2ff55d050585136531'. 2026-02-20T21:03:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 2026-02-20T21:03:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2026-02-20T21:03:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:changes and commit them, and you can discard any commits you make in this 2026-02-20T21:03:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:state without impacting any branches by switching back to a branch. 2026-02-20T21:03:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 2026-02-20T21:03:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:If you want to create a new branch to retain commits you create, you may 2026-02-20T21:03:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:do so (now or later) by using -c with the switch command. Example: 2026-02-20T21:03:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 2026-02-20T21:03:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: git switch -c 2026-02-20T21:03:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 2026-02-20T21:03:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Or undo this operation with: 2026-02-20T21:03:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 2026-02-20T21:03:37.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: git switch - 2026-02-20T21:03:37.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 2026-02-20T21:03:37.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2026-02-20T21:03:37.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 2026-02-20T21:03:37.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:HEAD is now at f7870454149 ceph-dev-pipeline: configure 2026-02-20T21:03:37.351 DEBUG:teuthology.orchestra.run.trial199:> cd -- /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror 2026-02-20T21:03:37.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2026-02-20T21:03:37.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-20T21:03:37.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2026-02-20T21:03:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2026-02-20T21:03:37.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2026-02-20T21:03:37.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-20T21:03:37.568 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-20T21:03:37.568 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2026-02-20T21:03:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-20T21:03:37.597 DEBUG:teuthology.orchestra.run.trial199:> set -ex 2026-02-20T21:03:37.597 DEBUG:teuthology.orchestra.run.trial199:> dd if=/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror of=/dev/stdout 2026-02-20T21:03:37.655 INFO:tasks.workunit:Running workunits matching rbd/rbd_mirror.sh on cluster1.client.mirror... 2026-02-20T21:03:37.658 INFO:tasks.workunit:Running workunit rbd/rbd_mirror.sh... 2026-02-20T21:03:37.658 DEBUG:teuthology.orchestra.run.trial199:workunit test rbd/rbd_mirror.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=f7870454149f22574086dd2ff55d050585136531 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS='' RBD_IMAGE_FEATURES=layering,exclusive-lock,object-map,fast-diff RBD_MIRROR_CONFIG_KEY=1 RBD_MIRROR_INSTANCES=4 RBD_MIRROR_MODE=snapshot RBD_MIRROR_USE_EXISTING_CLUSTER=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror.sh 2026-02-20T21:03:37.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 -gt 0 ']' 2026-02-20T21:03:37.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot '!=' snapshot ']' 2026-02-20T21:03:37.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror.sh 2026-02-20T21:03:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ . /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_helpers.sh 2026-02-20T21:03:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ RBD_MIRROR_INSTANCES=4 2026-02-20T21:03:37.718 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CLUSTER1=cluster1 2026-02-20T21:03:37.718 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CLUSTER2=cluster2 2026-02-20T21:03:37.718 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ PEER_CLUSTER_SUFFIX= 2026-02-20T21:03:37.718 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ POOL=mirror 2026-02-20T21:03:37.718 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ PARENT_POOL=mirror_parent 2026-02-20T21:03:37.718 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ NS1=ns1 2026-02-20T21:03:37.718 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ NS2=ns2 2026-02-20T21:03:37.718 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ TEMPDIR= 2026-02-20T21:03:37.718 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ID=mirror 2026-02-20T21:03:37.718 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ RBD_IMAGE_FEATURES=layering,exclusive-lock,object-map,fast-diff 2026-02-20T21:03:37.718 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ MIRROR_USER_ID_PREFIX=mirror. 2026-02-20T21:03:37.719 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ RBD_MIRROR_MODE=snapshot 2026-02-20T21:03:37.719 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ MIRROR_POOL_MODE=pool 2026-02-20T21:03:37.719 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' snapshot = snapshot ']' 2026-02-20T21:03:37.719 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ MIRROR_POOL_MODE=image 2026-02-20T21:03:37.719 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ export 'CEPH_ARGS=--id mirror' 2026-02-20T21:03:37.719 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS='--id mirror' 2026-02-20T21:03:37.719 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ LAST_MIRROR_INSTANCE=3 2026-02-20T21:03:37.719 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror.sh 2026-02-20T21:03:37.719 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ readlink -f /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/../../../src 2026-02-20T21:03:37.719 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror/src 2026-02-20T21:03:37.719 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_BIN=. 2026-02-20T21:03:37.720 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_SRC=. 2026-02-20T21:03:37.720 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' -e CMakeCache.txt ']' 2026-02-20T21:03:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 -gt 0 ']' 2026-02-20T21:03:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ setup 2026-02-20T21:03:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local c 2026-02-20T21:03:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ trap 'cleanup $?' INT TERM EXIT 2026-02-20T21:03:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ setup_tempdir 2026-02-20T21:03:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:03:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mktemp -d 2026-02-20T21:03:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ TEMPDIR=/tmp/tmp.2FjbyoErqN 2026-02-20T21:03:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -z 1 ']' 2026-02-20T21:03:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ setup_pools cluster1 cluster2 2026-02-20T21:03:37.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:03:37.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_cluster=cluster2 2026-02-20T21:03:37.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mon_map_file 2026-02-20T21:03:37.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mon_addr 2026-02-20T21:03:37.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local admin_key_file 2026-02-20T21:03:37.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local uuid 2026-02-20T21:03:37.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:03:37.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --cluster cluster1 osd pool create mirror 64 64 2026-02-20T21:03:38.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:pool 'mirror' already exists 2026-02-20T21:03:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:03:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --cluster cluster1 osd pool create mirror_parent 64 64 2026-02-20T21:03:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:pool 'mirror_parent' already exists 2026-02-20T21:03:39.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:03:39.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 pool init mirror 2026-02-20T21:03:42.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:03:42.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 pool init mirror_parent 2026-02-20T21:03:45.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 1 ']' 2026-02-20T21:03:45.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ PEER_CLUSTER_SUFFIX=-DNE 2026-02-20T21:03:45.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:03:45.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror pool enable --site-name cluster1-DNE mirror image 2026-02-20T21:03:45.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent image 2026-02-20T21:03:45.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 namespace create mirror/ns1 2026-02-20T21:03:45.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 namespace create mirror/ns2 2026-02-20T21:03:45.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 namespace create mirror_parent/ns1 2026-02-20T21:03:45.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 image 2026-02-20T21:03:45.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns2 image 2026-02-20T21:03:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent/ns1 image 2026-02-20T21:03:45.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -z ']' 2026-02-20T21:03:45.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -z 1 ']' 2026-02-20T21:03:45.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mon_map_file=/tmp/tmp.2FjbyoErqN/cluster2.monmap 2026-02-20T21:03:45.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:03:45.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --cluster cluster2 mon getmap 2026-02-20T21:03:45.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:got monmap epoch 1 2026-02-20T21:03:45.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ monmaptool --print /tmp/tmp.2FjbyoErqN/cluster2.monmap 2026-02-20T21:03:45.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -E 'mon\.' 2026-02-20T21:03:45.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ head -n 1 2026-02-20T21:03:45.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -E 's/^[0-9]+: ([^ ]+).+$/\1/' 2026-02-20T21:03:45.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -E 's/\/[0-9]+//g' 2026-02-20T21:03:45.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mon_addr='[v2:10.20.193.199:3300,v1:10.20.193.199:6789]' 2026-02-20T21:03:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_key_file=/tmp/tmp.2FjbyoErqN/cluster2.client.mirror.key 2026-02-20T21:03:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:03:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --cluster cluster2 auth get-key client.mirror 2026-02-20T21:03:45.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:03:45.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ peer_add cluster1 mirror client.mirror@cluster2-DNE '' --remote-mon-host '[v2:10.20.193.199:3300,v1:10.20.193.199:6789]' --remote-key-file /tmp/tmp.2FjbyoErqN/cluster2.client.mirror.key 2026-02-20T21:03:45.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:03:45.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:03:45.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:03:45.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:03:45.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local client_cluster=client.mirror@cluster2-DNE 2026-02-20T21:03:45.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:03:45.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_cluster=cluster2-DNE 2026-02-20T21:03:45.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local uuid_var_name 2026-02-20T21:03:45.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:03:45.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local error_code 2026-02-20T21:03:45.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local peer_uuid 2026-02-20T21:03:45.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 16 32 2026-02-20T21:03:45.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set +e 2026-02-20T21:03:45.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror client.mirror@cluster2-DNE --remote-mon-host '[v2:10.20.193.199:3300,v1:10.20.193.199:6789]' --remote-key-file /tmp/tmp.2FjbyoErqN/cluster2.client.mirror.key 2026-02-20T21:03:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ peer_uuid=051657c0-214b-4bab-85b7-d4f8366fff9d 2026-02-20T21:03:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ error_code=0 2026-02-20T21:03:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set -e 2026-02-20T21:03:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 -eq 17 ']' 2026-02-20T21:03:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 0 -eq 0 2026-02-20T21:03:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:03:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:03:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ peer_add cluster1 mirror_parent client.mirror@cluster2-DNE uuid 2026-02-20T21:03:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:03:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:03:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:03:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:03:45.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local client_cluster=client.mirror@cluster2-DNE 2026-02-20T21:03:45.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:03:45.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_cluster=cluster2-DNE 2026-02-20T21:03:45.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local uuid_var_name 2026-02-20T21:03:45.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n uuid ']' 2026-02-20T21:03:45.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ uuid_var_name=uuid 2026-02-20T21:03:45.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:03:45.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local error_code 2026-02-20T21:03:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local peer_uuid 2026-02-20T21:03:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 16 32 2026-02-20T21:03:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set +e 2026-02-20T21:03:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror_parent client.mirror@cluster2-DNE 2026-02-20T21:03:45.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ peer_uuid=09866fd0-f275-49ba-8515-82be1eebfde5 2026-02-20T21:03:45.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ error_code=0 2026-02-20T21:03:45.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set -e 2026-02-20T21:03:45.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 -eq 17 ']' 2026-02-20T21:03:45.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 0 -eq 0 2026-02-20T21:03:45.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n uuid ']' 2026-02-20T21:03:45.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval uuid=09866fd0-f275-49ba-8515-82be1eebfde5 2026-02-20T21:03:45.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ uuid=09866fd0-f275-49ba-8515-82be1eebfde5 2026-02-20T21:03:45.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:03:45.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:03:45.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror pool peer set mirror_parent 09866fd0-f275-49ba-8515-82be1eebfde5 mon-host '[v2:10.20.193.199:3300,v1:10.20.193.199:6789]' 2026-02-20T21:03:45.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:03:45.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror pool peer set mirror_parent 09866fd0-f275-49ba-8515-82be1eebfde5 key-file /tmp/tmp.2FjbyoErqN/cluster2.client.mirror.key 2026-02-20T21:03:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ setup_pools cluster2 cluster1 2026-02-20T21:03:45.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:03:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_cluster=cluster1 2026-02-20T21:03:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mon_map_file 2026-02-20T21:03:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mon_addr 2026-02-20T21:03:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local admin_key_file 2026-02-20T21:03:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local uuid 2026-02-20T21:03:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:03:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --cluster cluster2 osd pool create mirror 64 64 2026-02-20T21:03:46.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:pool 'mirror' already exists 2026-02-20T21:03:46.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:03:46.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --cluster cluster2 osd pool create mirror_parent 64 64 2026-02-20T21:03:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:pool 'mirror_parent' already exists 2026-02-20T21:03:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:03:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 pool init mirror 2026-02-20T21:03:50.824 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:03:50.824 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 pool init mirror_parent 2026-02-20T21:03:53.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 1 ']' 2026-02-20T21:03:53.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ PEER_CLUSTER_SUFFIX=-DNE 2026-02-20T21:03:53.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:03:53.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror pool enable --site-name cluster2-DNE mirror image 2026-02-20T21:03:53.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent image 2026-02-20T21:03:53.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 namespace create mirror/ns1 2026-02-20T21:03:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 namespace create mirror/ns2 2026-02-20T21:03:53.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 namespace create mirror_parent/ns1 2026-02-20T21:03:53.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 image 2026-02-20T21:03:53.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns2 image 2026-02-20T21:03:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent/ns1 image 2026-02-20T21:03:54.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -z ']' 2026-02-20T21:03:54.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -z 1 ']' 2026-02-20T21:03:54.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mon_map_file=/tmp/tmp.2FjbyoErqN/cluster1.monmap 2026-02-20T21:03:54.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:03:54.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --cluster cluster1 mon getmap 2026-02-20T21:03:54.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:got monmap epoch 1 2026-02-20T21:03:54.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ monmaptool --print /tmp/tmp.2FjbyoErqN/cluster1.monmap 2026-02-20T21:03:54.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -E 'mon\.' 2026-02-20T21:03:54.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ head -n 1 2026-02-20T21:03:54.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -E 's/^[0-9]+: ([^ ]+).+$/\1/' 2026-02-20T21:03:54.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -E 's/\/[0-9]+//g' 2026-02-20T21:03:54.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mon_addr='[v2:10.20.193.196:3300,v1:10.20.193.196:6789]' 2026-02-20T21:03:54.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_key_file=/tmp/tmp.2FjbyoErqN/cluster1.client.mirror.key 2026-02-20T21:03:54.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:03:54.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --cluster cluster1 auth get-key client.mirror 2026-02-20T21:03:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:03:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ peer_add cluster2 mirror client.mirror@cluster1-DNE '' --remote-mon-host '[v2:10.20.193.196:3300,v1:10.20.193.196:6789]' --remote-key-file /tmp/tmp.2FjbyoErqN/cluster1.client.mirror.key 2026-02-20T21:03:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:03:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:03:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:03:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:03:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local client_cluster=client.mirror@cluster1-DNE 2026-02-20T21:03:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:03:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_cluster=cluster1-DNE 2026-02-20T21:03:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local uuid_var_name 2026-02-20T21:03:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:03:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local error_code 2026-02-20T21:03:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local peer_uuid 2026-02-20T21:03:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 16 32 2026-02-20T21:03:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set +e 2026-02-20T21:03:54.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror client.mirror@cluster1-DNE --remote-mon-host '[v2:10.20.193.196:3300,v1:10.20.193.196:6789]' --remote-key-file /tmp/tmp.2FjbyoErqN/cluster1.client.mirror.key 2026-02-20T21:03:54.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ peer_uuid=8f9db7bf-ab51-45f3-a841-1009ca678b99 2026-02-20T21:03:54.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ error_code=0 2026-02-20T21:03:54.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set -e 2026-02-20T21:03:54.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 -eq 17 ']' 2026-02-20T21:03:54.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 0 -eq 0 2026-02-20T21:03:54.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:03:54.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:03:54.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ peer_add cluster2 mirror_parent client.mirror@cluster1-DNE uuid 2026-02-20T21:03:54.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:03:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:03:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:03:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:03:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local client_cluster=client.mirror@cluster1-DNE 2026-02-20T21:03:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:03:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_cluster=cluster1-DNE 2026-02-20T21:03:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local uuid_var_name 2026-02-20T21:03:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n uuid ']' 2026-02-20T21:03:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ uuid_var_name=uuid 2026-02-20T21:03:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:03:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local error_code 2026-02-20T21:03:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local peer_uuid 2026-02-20T21:03:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 16 32 2026-02-20T21:03:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set +e 2026-02-20T21:03:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror_parent client.mirror@cluster1-DNE 2026-02-20T21:03:54.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ peer_uuid=f3d83b73-cb25-4507-a075-2ec203560b78 2026-02-20T21:03:54.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ error_code=0 2026-02-20T21:03:54.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set -e 2026-02-20T21:03:54.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 -eq 17 ']' 2026-02-20T21:03:54.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 0 -eq 0 2026-02-20T21:03:54.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n uuid ']' 2026-02-20T21:03:54.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval uuid=f3d83b73-cb25-4507-a075-2ec203560b78 2026-02-20T21:03:54.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ uuid=f3d83b73-cb25-4507-a075-2ec203560b78 2026-02-20T21:03:54.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:03:54.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:03:54.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror pool peer set mirror_parent f3d83b73-cb25-4507-a075-2ec203560b78 mon-host '[v2:10.20.193.196:3300,v1:10.20.193.196:6789]' 2026-02-20T21:03:54.732 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:03:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror pool peer set mirror_parent f3d83b73-cb25-4507-a075-2ec203560b78 key-file /tmp/tmp.2FjbyoErqN/cluster1.client.mirror.key 2026-02-20T21:03:54.759 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:03:54.759 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: mirror from default namespace to non-default namespace' 2026-02-20T21:03:54.759 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:03:54.759 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:03:54.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:03:54 TEST: mirror from default namespace to non-default namespace 2026-02-20T21:03:54.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:03:54 TEST: mirror from default namespace to non-default namespace 2026-02-20T21:03:54.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirrors cluster1 2026-02-20T21:03:54.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:03:54.762 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:03:54.762 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:03:54.762 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:0 2026-02-20T21:03:54.762 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:03:54.762 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:03:54.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:03:54.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:03:54.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:03:54.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:03:54.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:03:54.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:03:54.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:03:54.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:03:54.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:03:54.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:03:54.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:03:54.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:03:54.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:03:54.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:03:54.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:1 2026-02-20T21:03:54.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:03:54.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:03:54.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:03:54.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:03:54.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:03:54.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:03:54.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:03:54.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:03:54.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:03:54.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:03:54.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:03:54.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:03:54.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:03:54.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:03:54.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:03:54.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:03:54.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:2 2026-02-20T21:03:54.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:03:54.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:03:54.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:03:54.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:2 2026-02-20T21:03:54.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:03:54.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:03:54.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:03:54.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=2 2026-02-20T21:03:54.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:03:54.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:03:54.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:03:54.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=2 2026-02-20T21:03:54.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:03:54.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:03:54.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:03:54.824 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:03:54.824 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:3 2026-02-20T21:03:54.824 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:03:54.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:03:54.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:03:54.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:3 2026-02-20T21:03:54.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:03:54.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:03:54.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:03:54.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=3 2026-02-20T21:03:54.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:03:54.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:03:54.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:03:54.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=3 2026-02-20T21:03:54.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:03:54.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:03:54.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:03:54.845 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image=test 2026-02-20T21:03:54.845 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror pool enable mirror init-only 2026-02-20T21:03:54.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror pool enable mirror init-only 2026-02-20T21:03:54.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror pool disable mirror/ns1 2026-02-20T21:03:54.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror pool disable mirror/ns1 2026-02-20T21:03:54.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror pool enable mirror image --remote-namespace ns1 2026-02-20T21:03:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 image --remote-namespace '' 2026-02-20T21:03:55.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image_and_enable_mirror cluster2 mirror test snapshot 2026-02-20T21:03:55.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:03:55.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:03:55.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:03:55.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:03:55.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:03:55.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:03:55.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:03:55.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:03:55.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:03:55.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image cluster2 mirror test 2026-02-20T21:03:55.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:03:55.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:03:55.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:03:55.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:03:55.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:03:55.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:03:55.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=128 2026-02-20T21:03:55.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:03:55.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,object-map,fast-diff mirror/test 2026-02-20T21:03:55.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:03:55.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:03:55.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:03:55.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:03:55.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror test snapshot 2026-02-20T21:03:55.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:03:55.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:03:55.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:03:55.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:03:55.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2026-02-20T21:03:55.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:03:55.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/test 2026-02-20T21:03:55.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test': 2026-02-20T21:03:55.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:03:55.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:03:55.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:03:55.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 10cc64283e8c 2026-02-20T21:03:55.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.10cc64283e8c 2026-02-20T21:03:55.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:03:55.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:03:55.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:03:55.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:03:55.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:03:55 2026 2026-02-20T21:03:55.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:03:55 2026 2026-02-20T21:03:55.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:03:55 2026 2026-02-20T21:03:55.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:03:55.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:03:55.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 61c6d423-64eb-4b02-897d-d7ab88532003 2026-02-20T21:03:55.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:03:55.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror/ns1 test 2026-02-20T21:03:55.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:03:55.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:03:55.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:03:55.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror/ns1 test started 2026-02-20T21:03:55.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:03:55.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:03:55.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:03:55.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:03:55.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:03:55.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:03:55.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:03:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror/ns1 test started 2026-02-20T21:03:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:03:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:03:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:03:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:03:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:03:56.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:03:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/ns1/test 2026-02-20T21:03:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:03:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:03:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:03:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:03:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:03:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:03:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:03:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:03:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:03:56.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:03:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:03:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test 2026-02-20T21:03:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:03:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:03:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:03:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:03:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:03:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:03:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:03:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:03:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:03:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:03:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:03:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:03:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:03:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:03:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:03:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:03:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:03:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:03:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:03:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:03:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:03:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:03:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:03:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:03:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:03:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:03:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:03:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:03:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:03:56.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:03:56.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:03:56.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:03:56.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:03:56.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test 2026-02-20T21:03:56.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-02-20T21:03:56.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:03:56.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:03:56.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:03:56.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns1/test 2026-02-20T21:03:56.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:03:56.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:03:56.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:03:56.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:03:56.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:03:56.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:03:56.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:03:56.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:03:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:03:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:03:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:03:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:03:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:03:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:03:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:03:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:03:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:03:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:03:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:03:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:03:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:03:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:03:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:03:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:03:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:03:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:03:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:03:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:03:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:03:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:03:56.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:03:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:03:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:03:56.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns1/test 2026-02-20T21:03:56.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-02-20T21:03:56.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:03:56.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:03:56.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:03:56.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns1/test 2026-02-20T21:03:56.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:03:56.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:03:56.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:03:56.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:03:56.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:03:56.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:03:56.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:03:56.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:03:56.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:03:56.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:03:56.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:03:56.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:03:56.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:03:56.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:03:56.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:03:56.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:03:56.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:03:56.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:03:56.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:03:56.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:03:56.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:03:56.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:03:56.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:03:56.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:03:56.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:03:56.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:03:56.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:03:56.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:03:56.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:03:56.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:03:56.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:03:56.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:03:56.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:03:56.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns1/test 2026-02-20T21:03:57.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-02-20T21:03:57.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:03:57.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:03:57.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:03:57.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns1/test 2026-02-20T21:03:57.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:03:57.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:03:57.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:03:57.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:03:57.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:03:57.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:03:57.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:03:57.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:03:57.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:03:57.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:03:57.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:03:57.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:03:57.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:03:57.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:03:57.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:03:57.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:03:57.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:03:57.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:03:57.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:03:57.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:03:57.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:03:57.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:03:57.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:03:57.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:03:57.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:03:57.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:03:57.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:03:57.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:03:57.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:03:57.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:03:57.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:03:57.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:03:57.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:03:57.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns1/test 2026-02-20T21:03:57.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-02-20T21:03:57.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:03:57.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:03:57.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:03:57.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 1 2026-02-20T21:03:58.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:03:58.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:03:58.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:03:58.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test 2026-02-20T21:03:58.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:03:58.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:03:58.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:03:58.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:03:58.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:03:58.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:03:58.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:03:58.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:03:58.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:03:58.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:03:58.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:03:58.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:03:58.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:03:58.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:03:58.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:03:58.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:03:58.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:03:58.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:03:58.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:03:58.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:03:58.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:03:58.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:03:58.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:03:58.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:03:58.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:03:58.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:03:58.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:03:58.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:03:58.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:03:58.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:03:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:03:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:03:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:03:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test 2026-02-20T21:03:58.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-02-20T21:03:58.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:03:58.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:03:58.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:03:58.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns1/test 2026-02-20T21:03:58.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:03:58.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:03:58.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:03:58.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:03:58.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:03:58.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:03:58.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:03:58.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:03:58.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:03:58.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:03:58.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:03:58.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:03:58.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:03:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:03:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:03:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:03:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:03:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:03:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:03:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:03:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:03:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:03:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:03:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:03:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:03:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:03:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:03:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:03:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:03:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:03:58.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:03:58.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:03:58.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:03:58.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns1/test 2026-02-20T21:03:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-02-20T21:03:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:03:58.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:03:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:03:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns1/test 2026-02-20T21:03:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:03:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:03:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:03:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:03:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:03:58.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:03:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:03:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:03:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:03:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:03:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:03:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:03:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:03:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:03:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:03:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:03:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:03:58.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:03:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:03:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:03:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:03:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:03:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:03:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:03:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:03:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:03:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:03:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:03:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:03:58.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:03:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:03:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:03:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:03:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns1/test 2026-02-20T21:03:58.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-02-20T21:03:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:03:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:03:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:03:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns1/test 2026-02-20T21:03:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:03:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:03:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:03:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:03:58.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:03:58.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:03:58.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:03:58.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:03:58.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:03:58.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:03:58.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:03:58.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:03:58.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:03:58.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:03:58.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:03:58.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:03:58.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:03:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:03:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:03:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:03:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:03:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:03:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:03:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:03:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:03:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:03:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:03:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:03:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:03:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:03:58.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:03:58.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:03:58.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:03:58.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns1/test 2026-02-20T21:03:58.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-02-20T21:03:58.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:03:58.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:03:58.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:03:58.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 2 2026-02-20T21:04:00.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:04:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:04:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test 2026-02-20T21:04:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:04:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:04:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:04:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:00.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:00.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:04:00.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:04:00.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test 2026-02-20T21:04:00.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-02-20T21:04:00.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:04:00.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:00.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:00.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns1/test 2026-02-20T21:04:00.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:04:00.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:00.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:00.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:04:00.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:04:00.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:00.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:00.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:00.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:00.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:00.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:00.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:00.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:04:00.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:00.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:04:00.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:04:00.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:00.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:00.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:04:00.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:04:00.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:00.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:00.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:00.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:00.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:00.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:00.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:00.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:04:00.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:04:00.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:04:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:00.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:00.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns1/test 2026-02-20T21:04:00.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-02-20T21:04:00.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:04:00.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns1/test 2026-02-20T21:04:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:04:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:04:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:04:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:04:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:04:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:04:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:04:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:04:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:04:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:04:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:04:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:04:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:04:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:04:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:00.434 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:00.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns1/test 2026-02-20T21:04:00.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-02-20T21:04:00.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:04:00.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns1/test 2026-02-20T21:04:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:04:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:04:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:04:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:04:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:04:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:04:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:04:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:04:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:04:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:04:00.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:00.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:00.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:00.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:04:00.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:00.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:00.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:00.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:04:00.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:04:00.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:04:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:00.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns1/test 2026-02-20T21:04:00.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-02-20T21:04:00.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:04:00.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:00.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:00.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 4 2026-02-20T21:04:04.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:04:04.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:04:04.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test 2026-02-20T21:04:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:04:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:04.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:04.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:04.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:04.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:04.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:04:04.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:04.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:04.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:04.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:04:04.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:04.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:04.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:04.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:04:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:04:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:04.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:04.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:04.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:04.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test 2026-02-20T21:04:04.593 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:04:04.593 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:04:04.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:04:04.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:04:04.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:04:04.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:04:04.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:04:04.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:04.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test 100 2026-02-20T21:04:04.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:04.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:04.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:04.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:04:04.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:04:04.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:04:04.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:04:04.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:04:04.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:04:04.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:04:04.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5263.23 bytes/sec: 21 MiB/s 2026-02-20T21:04:04.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror/ns1 mirror test 2026-02-20T21:04:04.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:04:04.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:04.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror/ns1 2026-02-20T21:04:04.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:04:04.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:04.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:04:04.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:04:04.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:04:04.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:04.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:04.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:04.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:04:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 4 2026-02-20T21:04:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror/ns1 mirror test 2026-02-20T21:04:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:04:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror/ns1 2026-02-20T21:04:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:04:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:04:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:04:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:04:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:04:05.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror/ns1-test.status 2026-02-20T21:04:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror/ns1-test.status 2026-02-20T21:04:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:04:05.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.status 2026-02-20T21:04:05.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:04:05.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:05.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:05.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:05.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:04:05.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:04:05.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:04:05.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:04:05.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=4 2026-02-20T21:04:05.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:04:05.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:04:05.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:04:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.status 2026-02-20T21:04:05.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:05.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:04:05.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:05.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.status 2026-02-20T21:04:05.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test --format xml 2026-02-20T21:04:05.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:04:05.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:04:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=3 2026-02-20T21:04:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 4 = 3 2026-02-20T21:04:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:04:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.4 2026-02-20T21:04:05.730 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.status 2026-02-20T21:04:05.730 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:05.730 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:04:05.730 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:05.730 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.status 2026-02-20T21:04:05.730 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test --format xml 2026-02-20T21:04:05.730 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:04:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:04:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=3 2026-02-20T21:04:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 4 = 3 2026-02-20T21:04:05.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:04:05.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.8 2026-02-20T21:04:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.status 2026-02-20T21:04:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:04:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.status 2026-02-20T21:04:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test --format xml 2026-02-20T21:04:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:04:06.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:04:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=4 2026-02-20T21:04:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 4 = 4 2026-02-20T21:04:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror/ns1 test 2026-02-20T21:04:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:04:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:04:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror/ns1 test up+replaying 2026-02-20T21:04:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:04:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:07.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror/ns1 test up+replaying '' '' 2026-02-20T21:04:07.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:07.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:04:07.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:07.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:07.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:07.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:07.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:07.601 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:07.601 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/ns1/test 2026-02-20T21:04:07.632 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:04:07.632 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 61c6d423-64eb-4b02-897d-d7ab88532003 2026-02-20T21:04:07.632 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:04:07.632 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621435,"replay_state":"idle"} 2026-02-20T21:04:07.632 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:04:07.632 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:04:01' 2026-02-20T21:04:07.633 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:04:07.633 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:04:07.633 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:04:07.634 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621435,"replay_state":"idle"} 2026-02-20T21:04:07.634 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:04:07.634 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:04:07.635 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:04:07.635 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:04:07.635 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:04:07.636 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:04:07.636 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:04:07.636 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:04:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:01' 2026-02-20T21:04:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror/ns1 test up+replaying '2026-02-20 21:04:01' 2026-02-20T21:04:07.638 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:07.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:04:07.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:07.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:07.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:04:01' 2026-02-20T21:04:07.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:07.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:07.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror/ns1 --verbose --format xml 2026-02-20T21:04:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING142794284mirror.0trial19920.2.0-677-gf7870454trueOK42864290mirror.1trial19920.2.0-677-gf7870454falseOK42924300mirror.2trial19920.2.0-677-gf7870454falseOK43024309mirror.3trial19920.2.0-677-gf7870454falseOKtest61c6d423-64eb-4b02-897d-d7ab88532003up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621435,"replay_state":"idle"}42794284mirror.0trial1992026-02-20 21:04:01' 2026-02-20T21:04:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:04:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:04:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:01' 2026-02-20T21:04:07.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:04:07.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:04:07.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:04:07.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:04:07.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:04:07.676 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:07.676 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:07.676 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror/ns1 mirror test 2026-02-20T21:04:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:04:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:04:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror/ns1 2026-02-20T21:04:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:04:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.export 2026-02-20T21:04:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.export 2026-02-20T21:04:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:04:07.678 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export 2026-02-20T21:04:07.678 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror/ns1-test.export 2026-02-20T21:04:07.678 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror/ns1-test.export 2026-02-20T21:04:07.678 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:04:07.679 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.export 2026-02-20T21:04:07.679 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.export 2026-02-20T21:04:07.679 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export 2026-02-20T21:04:07.759 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:04:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/ns1/test /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.export 2026-02-20T21:04:07.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:04:07.849 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.export 2026-02-20T21:04:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.export 2026-02-20T21:04:07.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:07.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster2 mirror test 2026-02-20T21:04:07.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:07.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:07.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:07.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:04:07.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:04:07.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster2 mirror test 2026-02-20T21:04:07.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:07.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:07.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:07.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 snap purge mirror/test 2026-02-20T21:04:07.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rm mirror/test 2026-02-20T21:04:09.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:04:09.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:09.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror/ns1 test deleted 2026-02-20T21:04:09.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:09.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:04:09.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:09.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=deleted 2026-02-20T21:04:09.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:04:09.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:04:09.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:04:09.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror/ns1 test 2026-02-20T21:04:09.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:09.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror/ns1 2026-02-20T21:04:09.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:04:09.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/ns1/test 2026-02-20T21:04:09.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:04:09.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:04:09.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image_id= 2026-02-20T21:04:09.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:04:09.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:04:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror/ns1 test deleted '' 2026-02-20T21:04:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:04:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=deleted 2026-02-20T21:04:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:04:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:04:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:04:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror/ns1 test 2026-02-20T21:04:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror/ns1 2026-02-20T21:04:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:04:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/ns1/test 2026-02-20T21:04:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:04:09.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:04:09.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id= 2026-02-20T21:04:09.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:04:09.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test deleted = deleted 2026-02-20T21:04:09.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:09.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror pool disable mirror/ns1 2026-02-20T21:04:09.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror pool enable mirror init-only 2026-02-20T21:04:09.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: mirror from non-default namespace to default namespace' 2026-02-20T21:04:09.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:04:09.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:04:09.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:04:09 TEST: mirror from non-default namespace to default namespace 2026-02-20T21:04:09.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:04:09 TEST: mirror from non-default namespace to default namespace 2026-02-20T21:04:09.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 image --remote-namespace '' 2026-02-20T21:04:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror pool enable mirror image --remote-namespace ns1 2026-02-20T21:04:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image_and_enable_mirror cluster2 mirror/ns1 test snapshot 2026-02-20T21:04:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:04:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:04:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:04:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image cluster2 mirror/ns1 test 2026-02-20T21:04:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:04:09.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:09.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:09.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:09.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=128 2026-02-20T21:04:09.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:04:09.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,object-map,fast-diff mirror/ns1/test 2026-02-20T21:04:09.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror/ns1 2026-02-20T21:04:09.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:04:09.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:04:09.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:09.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror/ns1 test snapshot 2026-02-20T21:04:09.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:09.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:04:09.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:09.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:04:09.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/ns1/test snapshot 2026-02-20T21:04:10.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:04:10.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/ns1/test 2026-02-20T21:04:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test': 2026-02-20T21:04:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:04:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:04:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:04:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 10ed936e0d63 2026-02-20T21:04:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.10ed936e0d63 2026-02-20T21:04:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:04:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:04:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:04:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:04:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:04:09 2026 2026-02-20T21:04:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:04:09 2026 2026-02-20T21:04:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:04:09 2026 2026-02-20T21:04:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:04:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:04:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: e3f41246-917e-4f0e-b3d2-e01d7e91318a 2026-02-20T21:04:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:04:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:04:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:04:10.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:04:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:04:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:10.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:11.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:04:11.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:04:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:04:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:04:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:04:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:04:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:04:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:04:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:04:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:04:11.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:04:11.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:04:11.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:11.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:04:11.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:04:11.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:11.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:11.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:11.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:11.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:11.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:04:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:04:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:11.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:11.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:11.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:11.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:11.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:04:11.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:04:11.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:11.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:04:11.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:04:11.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:04:11.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:11.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:11.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:04:11.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:04:11.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:11.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:11.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:04:11.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:04:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:04:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:04:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:04:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:04:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:04:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:04:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:04:11.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:04:11.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:11.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:11.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:11.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:04:11.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:04:11.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:04:11.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:11.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:11.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:04:11.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:04:11.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:11.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:11.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:04:11.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:04:11.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:11.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:11.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:11.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:04:11.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:11.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:11.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:11.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:04:11.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:04:11.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:04:11.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:04:11.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:11.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:11.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:04:11.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:04:11.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:11.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:11.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:11.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:04:11.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:11.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:11.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:11.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:04:11.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:04:11.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:04:11.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:11.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:11.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:11.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:04:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:04:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:04:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:04:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:04:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:04:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:04:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:04:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:04:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:04:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:04:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:04:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:04:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:04:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:04:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:04:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:04:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:04:11.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:11.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:11.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:11.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:04:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:04:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:04:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:11.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:11.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 1 2026-02-20T21:04:12.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:04:12.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:04:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:04:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:04:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:04:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:04:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:04:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:04:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:12.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:12.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:12.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:12.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:04:12.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:04:12.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:04:12.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:04:12.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:04:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:04:12.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:04:12.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:04:12.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:12.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror/ns1 test 100 2026-02-20T21:04:12.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:04:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:04:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:04:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:04:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:04:12.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/ns1/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:04:12.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:04:12.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:04:12.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5882.43 bytes/sec: 23 MiB/s 2026-02-20T21:04:12.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror/ns1 test 2026-02-20T21:04:12.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:04:12.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:12.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:04:12.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror/ns1 2026-02-20T21:04:12.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:12.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:04:12.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:04:12.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror/ns1 test 2026-02-20T21:04:12.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:12.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:04:12.428 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:12.428 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/ns1/test 2026-02-20T21:04:13.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 6 2026-02-20T21:04:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror/ns1 test 2026-02-20T21:04:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:04:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:04:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror/ns1 2026-02-20T21:04:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:13.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror/ns1-test.status 2026-02-20T21:04:13.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror/ns1-test.status 2026-02-20T21:04:13.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:04:13.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test.status 2026-02-20T21:04:13.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:04:13.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:04:13.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:04:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:04:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror/ns1 test /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test.status 2026-02-20T21:04:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:04:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test.status 2026-02-20T21:04:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/ns1/test --format xml 2026-02-20T21:04:13.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:04:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:04:13.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=6 2026-02-20T21:04:13.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:04:13.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:04:13.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:04:13.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:04:13.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:13.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:13.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:13.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:04:13.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:04:13.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:04:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:04:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=5 2026-02-20T21:04:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 6 = 5 2026-02-20T21:04:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:04:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.4 2026-02-20T21:04:13.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:04:13.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:13.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:13.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:13.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:04:13.737 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:04:13.737 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:04:13.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:04:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=5 2026-02-20T21:04:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 6 = 5 2026-02-20T21:04:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:04:13.771 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.8 2026-02-20T21:04:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:04:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:04:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:04:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:04:14.603 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:04:14.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=6 2026-02-20T21:04:14.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 6 = 6 2026-02-20T21:04:14.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:14.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-20T21:04:14.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:14.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:14.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:14.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:04:14.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:04:14.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:14.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:14.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:14.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:14.607 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:14.607 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:14.607 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:14.607 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:04:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:04:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:04:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: e3f41246-917e-4f0e-b3d2-e01d7e91318a 2026-02-20T21:04:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:04:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621450,"replay_state":"idle"} 2026-02-20T21:04:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:04:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:04:12' 2026-02-20T21:04:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:04:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:04:15.640 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:04:15.640 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621450,"replay_state":"idle"} 2026-02-20T21:04:15.641 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:04:15.641 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:04:15.641 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:04:15.642 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:04:15.642 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:04:15.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:04:15.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:04:15.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:04:15.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:12' 2026-02-20T21:04:15.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:04:12' 2026-02-20T21:04:15.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:15.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:15.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:15.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:15.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:04:12' 2026-02-20T21:04:15.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:15.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:15.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:04:15.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING142794284mirror.0trial19920.2.0-677-gf7870454trueOK42864290mirror.1trial19920.2.0-677-gf7870454falseOK42924300mirror.2trial19920.2.0-677-gf7870454falseOK43024309mirror.3trial19920.2.0-677-gf7870454falseOKteste3f41246-917e-4f0e-b3d2-e01d7e91318aup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621450,"replay_state":"idle"}42794284mirror.0trial1992026-02-20 21:04:12' 2026-02-20T21:04:15.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:04:15.678 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:04:15.679 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:12' 2026-02-20T21:04:15.679 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:04:15.680 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:04:15.681 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:04:15.681 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:04:15.681 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:04:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror mirror/ns1 test 2026-02-20T21:04:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:04:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:04:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:04:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror/ns1 2026-02-20T21:04:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:15.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror/ns1-test.export 2026-02-20T21:04:15.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror/ns1-test.export 2026-02-20T21:04:15.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:04:15.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test.export 2026-02-20T21:04:15.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.export 2026-02-20T21:04:15.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.export 2026-02-20T21:04:15.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:04:15.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:04:15.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:04:15.686 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/ns1/test /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test.export 2026-02-20T21:04:15.771 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:04:15.776 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:04:15.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:04:15.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:04:15.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:04:15.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:15.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster2 mirror/ns1 test 2026-02-20T21:04:15.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:15.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:04:15.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:15.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:04:15.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:04:15.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster2 mirror/ns1 test 2026-02-20T21:04:15.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:15.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:04:15.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:15.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 snap purge mirror/ns1/test 2026-02-20T21:04:15.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rm mirror/ns1/test 2026-02-20T21:04:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:04:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror test deleted 2026-02-20T21:04:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=deleted 2026-02-20T21:04:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:04:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:04:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:04:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:04:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:04:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:04:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:04:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:04:17.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:04:17.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image_id= 2026-02-20T21:04:17.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:04:17.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:04:17.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test deleted '' 2026-02-20T21:04:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=deleted 2026-02-20T21:04:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:04:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:04:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:04:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:04:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:17.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:04:17.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:04:17.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:04:17.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:04:17.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:04:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id= 2026-02-20T21:04:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:04:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test deleted = deleted 2026-02-20T21:04:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror pool enable mirror init-only 2026-02-20T21:04:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror pool disable mirror/ns1 2026-02-20T21:04:17.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror pool enable mirror image 2026-02-20T21:04:17.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror pool enable mirror image 2026-02-20T21:04:17.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 image 2026-02-20T21:04:17.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 image 2026-02-20T21:04:17.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: add image and test replay' 2026-02-20T21:04:17.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:04:17.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:04:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:04:17 TEST: add image and test replay 2026-02-20T21:04:17.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:04:17 TEST: add image and test replay 2026-02-20T21:04:17.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image_and_enable_mirror cluster2 mirror test snapshot 2026-02-20T21:04:17.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:17.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:17.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:17.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:17.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:17.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:17.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:04:17.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:04:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image cluster2 mirror test 2026-02-20T21:04:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=128 2026-02-20T21:04:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:04:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,object-map,fast-diff mirror/test 2026-02-20T21:04:17.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:04:17.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:17.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:04:17.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:17.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror test snapshot 2026-02-20T21:04:17.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:17.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:17.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:17.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:04:17.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2026-02-20T21:04:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:04:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/test 2026-02-20T21:04:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test': 2026-02-20T21:04:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:04:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:04:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:04:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 11112aa1b418 2026-02-20T21:04:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.11112aa1b418 2026-02-20T21:04:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:04:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:04:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:04:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:04:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:04:17 2026 2026-02-20T21:04:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:04:17 2026 2026-02-20T21:04:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:04:17 2026 2026-02-20T21:04:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:04:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:04:18.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: c5fdae76-300b-4ccc-baee-3e7deda3028d 2026-02-20T21:04:18.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:04:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_image_meta cluster2 mirror test key1 value1 2026-02-20T21:04:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local key=key1 2026-02-20T21:04:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=value1 2026-02-20T21:04:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 image-meta set mirror/test key1 value1 2026-02-20T21:04:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_image_meta cluster2 mirror test key2 value2 2026-02-20T21:04:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:18.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:18.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local key=key2 2026-02-20T21:04:18.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=value2 2026-02-20T21:04:18.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 image-meta set mirror/test key2 value2 2026-02-20T21:04:18.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:04:18.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:18.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:18.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:18.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:04:18.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:18.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:18.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:18.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:04:18.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:04:18.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:18.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:04:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:04:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:04:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:04:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:04:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:04:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:04:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:04:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:04:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:04:19.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:04:19.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:04:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:04:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:04:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:04:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:04:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:04:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:04:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:19.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:19.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:19.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:19.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:04:19.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:04:19.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:04:19.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:19.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:19.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:04:19.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:04:19.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:19.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:19.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:04:19.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:04:19.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:19.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:19.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:19.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:19.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:19.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:19.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:19.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:04:19.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:19.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:04:19.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:04:19.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:19.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:19.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:04:19.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:04:19.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:19.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:19.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:19.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:19.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:19.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:19.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:19.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:04:19.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:04:19.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:04:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:19.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:19.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:19.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:04:19.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:04:19.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:04:19.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:19.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:19.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:04:19.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:04:19.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:19.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:19.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:04:19.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:04:19.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:19.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:19.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:19.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:04:19.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:19.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:19.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:19.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:04:19.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:04:19.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:04:19.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:04:19.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:19.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:19.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:04:19.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:04:19.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:19.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:19.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:19.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:04:19.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:19.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:19.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:19.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:04:19.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:04:19.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:04:19.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:19.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:19.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:19.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:04:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:04:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:04:19.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:19.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:19.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:04:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:04:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:04:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:04:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:04:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:04:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:04:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:04:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:04:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:04:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:04:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:04:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:04:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:04:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:04:19.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:19.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:19.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:19.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:04:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:04:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:04:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:19.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:19.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 1 2026-02-20T21:04:20.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:04:20.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:04:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:04:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:04:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:04:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:04:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:04:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:04:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:20.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:04:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:04:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:04:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:04:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:04:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:04:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:04:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:04:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:04:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:04:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:04:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:04:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:04:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:04:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:04:20.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:20.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:20.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:20.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:04:20.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:04:20.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:04:20.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:20.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:20.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:04:20.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:04:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:04:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:04:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:04:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:20.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:04:20.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:04:20.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:04:20.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:04:20.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:20.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:20.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:04:20.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:04:20.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:20.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:20.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:20.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:04:20.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:20.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:20.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:20.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:04:20.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:04:20.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:04:20.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:20.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:20.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:20.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:04:20.523 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:04:20.523 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:04:20.523 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:20.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:20.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:04:20.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:04:20.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:20.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:20.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:04:20.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:04:20.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:20.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:20.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:20.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:04:20.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:04:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:04:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:04:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:04:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:04:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:04:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:20.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:04:20.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:20.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:20.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:20.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:04:20.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:04:20.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:04:20.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:20.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:20.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:20.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:04:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:04:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:04:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:20.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:20.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 2 2026-02-20T21:04:22.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:04:22.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:04:22.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:22.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:04:22.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:04:22.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:22.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:22.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:22.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:22.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:22.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:22.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:22.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:22.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:22.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:22.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:04:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:04:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:04:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:04:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:22.593 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:22.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:22.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:22.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:04:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:04:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:04:22.641 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:04:22.641 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:04:22.641 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:04:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:04:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:04:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test 100 2026-02-20T21:04:22.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:22.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:22.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:22.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:04:22.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:04:22.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:04:22.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:04:22.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:04:22.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:04:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:04:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5882.43 bytes/sec: 23 MiB/s 2026-02-20T21:04:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:04:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:04:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:04:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:04:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:04:22.695 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:04:22.695 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:04:22.695 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:22.695 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:22.695 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:22.695 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:04:22.805 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 8 2026-02-20T21:04:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:04:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:04:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:04:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:04:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:04:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:04:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:04:22.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:04:22.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:04:22.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:04:22.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:04:22.815 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:04:22.815 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:04:22.815 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:22.815 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:22.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:22.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:04:22.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:04:22.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:04:22.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:04:22.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=8 2026-02-20T21:04:22.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:04:22.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:04:22.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:04:23.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:04:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:04:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:04:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:04:23.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:04:23.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=7 2026-02-20T21:04:23.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 8 = 7 2026-02-20T21:04:23.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:04:23.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.4 2026-02-20T21:04:23.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:04:23.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:23.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:23.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:23.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:04:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:04:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:04:23.507 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:04:23.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=7 2026-02-20T21:04:23.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 8 = 7 2026-02-20T21:04:23.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:04:23.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.8 2026-02-20T21:04:24.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:04:24.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:24.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:24.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:24.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:04:24.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:04:24.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:04:24.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:04:24.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=8 2026-02-20T21:04:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 8 = 8 2026-02-20T21:04:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-20T21:04:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:04:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:04:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:24.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:24.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:24.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:24.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:24.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:04:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:04:25.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:04:25.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: c5fdae76-300b-4ccc-baee-3e7deda3028d 2026-02-20T21:04:25.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:04:25.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621458,"replay_state":"idle"} 2026-02-20T21:04:25.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:04:25.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:04:21' 2026-02-20T21:04:25.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:04:25.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:04:25.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:04:25.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621458,"replay_state":"idle"} 2026-02-20T21:04:25.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:04:25.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:04:25.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:04:25.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:04:25.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:04:25.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:04:25.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:04:25.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:04:25.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:21' 2026-02-20T21:04:25.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:04:21' 2026-02-20T21:04:25.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:25.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:25.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:25.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:25.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:04:21' 2026-02-20T21:04:25.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:25.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:25.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:04:25.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING142794284mirror.0trial19920.2.0-677-gf7870454trueOK42864290mirror.1trial19920.2.0-677-gf7870454falseOK42924300mirror.2trial19920.2.0-677-gf7870454falseOK43024309mirror.3trial19920.2.0-677-gf7870454falseOKtestc5fdae76-300b-4ccc-baee-3e7deda3028dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621458,"replay_state":"idle"}42794284mirror.0trial1992026-02-20 21:04:21' 2026-02-20T21:04:25.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:04:25.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:04:25.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:21' 2026-02-20T21:04:25.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:04:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:04:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:04:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:04:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:04:25.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:25.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:25.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -z '' ']' 2026-02-20T21:04:25.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test down+unknown 2026-02-20T21:04:25.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:25.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:25.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:25.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=down+unknown 2026-02-20T21:04:25.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:25.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:25.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:25.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test down+unknown '' '' 2026-02-20T21:04:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=down+unknown 2026-02-20T21:04:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:26.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:26.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:04:26.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:04:26.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: c5fdae76-300b-4ccc-baee-3e7deda3028d 2026-02-20T21:04:26.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: down+unknown 2026-02-20T21:04:26.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: status not found 2026-02-20T21:04:26.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20T21:04:26.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:04:26.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:04:26.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:04:26.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621458,"replay_state":"idle"} 2026-02-20T21:04:26.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:04:21 2026-02-20T21:04:26.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:04:26.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 7 .mirror.primary.c5fdae76-300b-4ccc-baee-3e7deda3028d.ac92e6f3-a395-40a7-a286-d1ccb1203d73 (peer_uuids:[]) 2026-02-20T21:04:26.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 8 .mirror.primary.c5fdae76-300b-4ccc-baee-3e7deda3028d.3b02e961-75c7-4135-9b2b-dcfed7f8cdce (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:04:26.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*down+unknown' 2026-02-20T21:04:26.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: down+unknown 2026-02-20T21:04:26.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:04:26.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: status not found 2026-02-20T21:04:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:04:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo down+unknown 2026-02-20T21:04:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:04:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: ' 2026-02-20T21:04:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:04:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:04:26.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update= 2026-02-20T21:04:26.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test down+unknown '' 2026-02-20T21:04:26.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:26.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:26.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:26.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=down+unknown 2026-02-20T21:04:26.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local prev_last_update= 2026-02-20T21:04:26.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:26.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:26.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:04:26.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='UNKNOWNUNKNOWNOK1testc5fdae76-300b-4ccc-baee-3e7deda3028ddown+unknownstatus not foundcluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621458,"replay_state":"idle"}2026-02-20 21:04:21' 2026-02-20T21:04:26.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:04:26.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:04:26.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update= 2026-02-20T21:04:26.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:04:26.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=down+unknown 2026-02-20T21:04:26.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo down+unknown 2026-02-20T21:04:26.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep down+unknown 2026-02-20T21:04:26.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:down+unknown 2026-02-20T21:04:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror mirror test 2026-02-20T21:04:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:04:26.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:04:26.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:26.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:04:26.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:04:26.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:26.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.export 2026-02-20T21:04:26.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.export 2026-02-20T21:04:26.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:04:26.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export 2026-02-20T21:04:26.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.export 2026-02-20T21:04:26.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.export 2026-02-20T21:04:26.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:04:26.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:04:26.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:04:26.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export 2026-02-20T21:04:26.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:04:26.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:04:26.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:04:26.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:04:26.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:04:26.762 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:26.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_image_meta cluster1 mirror test key1 value1 2026-02-20T21:04:26.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:26.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:26.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:26.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local key=key1 2026-02-20T21:04:26.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local value=value1 2026-02-20T21:04:26.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 image-meta get mirror/test key1 2026-02-20T21:04:26.792 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test value1 = value1 2026-02-20T21:04:26.792 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_image_meta cluster1 mirror test key2 value2 2026-02-20T21:04:26.792 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:26.792 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:26.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:26.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local key=key2 2026-02-20T21:04:26.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local value=value2 2026-02-20T21:04:26.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 image-meta get mirror/test key2 2026-02-20T21:04:26.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test value2 = value2 2026-02-20T21:04:26.821 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: stop mirror, add image, start mirror and test replay' 2026-02-20T21:04:26.821 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:04:26.821 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:04:26.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:04:26 TEST: stop mirror, add image, start mirror and test replay 2026-02-20T21:04:26.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:04:26 TEST: stop mirror, add image, start mirror and test replay 2026-02-20T21:04:26.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirrors cluster1 2026-02-20T21:04:26.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:26.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:04:26.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:04:26.824 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:26.824 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:0 '' 2026-02-20T21:04:26.824 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:04:26.824 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:04:26.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:04:26.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:04:26.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:0 2026-02-20T21:04:26.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:0 2026-02-20T21:04:26.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:26.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:26.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:04:26.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:04:26.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:26.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:26.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:26.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:26.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:04:26.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:04:26.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:04:26.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:04:26.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-20T21:04:26.835 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:04:26.835 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:04:26.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=48498 2026-02-20T21:04:26.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 48498 ']' 2026-02-20T21:04:26.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 48498 2026-02-20T21:04:26.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:04:26.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:04:26.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:04:26.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=48498 '$2 == pid {print; exit 1}' 2026-02-20T21:04:26.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:04:26.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:04:26.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=48498 '$2 == pid {print; exit 1}' 2026-02-20T21:04:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:04:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:26.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:26.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:26.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:26.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-20T21:04:26.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:04:26.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:04:26.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:26.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:26.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:26.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:26.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:26.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:26.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:26.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:26.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:26.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:26.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:26.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:26.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:26.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:0 2026-02-20T21:04:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:04:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-20T21:04:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:04:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:04:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:1 '' 2026-02-20T21:04:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:04:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:04:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:04:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:04:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:1 2026-02-20T21:04:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:1 2026-02-20T21:04:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:04:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:04:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:04:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:04:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:04:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:04:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:04:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-20T21:04:26.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:04:26.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:04:26.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=48542 2026-02-20T21:04:26.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 48542 ']' 2026-02-20T21:04:26.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 48542 2026-02-20T21:04:26.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:04:26.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:04:26.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:04:26.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=48542 '$2 == pid {print; exit 1}' 2026-02-20T21:04:27.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:04:27.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:04:27.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=48542 '$2 == pid {print; exit 1}' 2026-02-20T21:04:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:04:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:04:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:04:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:04:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:27.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:04:27.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:27.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:04:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-20T21:04:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:04:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:04:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:04:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:04:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:04:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:04:27.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:27.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:27.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:1 2026-02-20T21:04:27.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:04:27.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:27.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:04:27.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:04:27.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:27.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:27.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:27.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:27.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:27.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:27.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:27.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:04:27.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:27.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-20T21:04:27.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:04:27.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:04:27.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:27.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:2 '' 2026-02-20T21:04:27.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:04:27.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:04:27.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:04:27.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:04:27.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:2 2026-02-20T21:04:27.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:2 2026-02-20T21:04:27.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:27.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:04:27.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:04:27.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:04:27.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:27.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:27.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:04:27.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:27.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:04:27.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:04:27.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:04:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:04:27.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-20T21:04:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:04:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:04:27.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=48602 2026-02-20T21:04:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 48602 ']' 2026-02-20T21:04:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 48602 2026-02-20T21:04:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:04:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:04:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:04:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=48602 '$2 == pid {print; exit 1}' 2026-02-20T21:04:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:04:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:04:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=48602 '$2 == pid {print; exit 1}' 2026-02-20T21:04:27.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:04:27.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:04:27.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:27.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:27.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:04:27.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:04:27.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:27.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:27.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:27.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:04:27.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:27.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:27.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:27.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:04:27.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:04:27.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:04:27.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:27.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-20T21:04:27.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:04:27.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:04:27.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:27.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:04:27.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:04:27.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:27.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:27.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:27.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:04:27.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:27.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:04:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:04:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:04:27.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:27.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:2 2026-02-20T21:04:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:04:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:04:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:04:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:04:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:04:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:04:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-20T21:04:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:04:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:04:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:3 '' 2026-02-20T21:04:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:04:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:04:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:04:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:04:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:3 2026-02-20T21:04:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:3 2026-02-20T21:04:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:04:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:04:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:04:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:04:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:04:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:04:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:04:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:04:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-20T21:04:27.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:04:27.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:04:27.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=48680 2026-02-20T21:04:27.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 48680 ']' 2026-02-20T21:04:27.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 48680 2026-02-20T21:04:27.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:04:27.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:04:27.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:04:27.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=48680 '$2 == pid {print; exit 1}' 2026-02-20T21:04:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:04:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:04:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=48680 '$2 == pid {print; exit 1}' 2026-02-20T21:04:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:04:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:04:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:04:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:04:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:04:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:04:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:04:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:04:27.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:27.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-20T21:04:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:04:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:04:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:04:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:04:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:04:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:04:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:04:27.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:04:27.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:27.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:27.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:3 2026-02-20T21:04:27.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:04:27.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:27.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:04:27.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:04:27.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:27.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:27.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:27.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:04:27.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:27.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:27.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:27.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:04:27.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:04:27.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-20T21:04:27.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:04:27.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:04:27.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image1=test1 2026-02-20T21:04:27.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image_and_enable_mirror cluster2 mirror test1 snapshot 2026-02-20T21:04:27.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:27.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:27.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:27.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:27.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:27.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:27.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:04:27.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:04:27.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:27.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image cluster2 mirror test1 2026-02-20T21:04:27.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:27.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:27.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:27.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:27.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:27.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:27.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=128 2026-02-20T21:04:27.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:04:27.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,object-map,fast-diff mirror/test1 2026-02-20T21:04:27.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:04:27.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:27.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:04:27.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:27.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror test1 snapshot 2026-02-20T21:04:27.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:27.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:27.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:27.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:04:27.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test1 snapshot 2026-02-20T21:04:28.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:04:28.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/test1 2026-02-20T21:04:28.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test1': 2026-02-20T21:04:28.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:04:28.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:04:28.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:04:28.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 11326066b542 2026-02-20T21:04:28.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.11326066b542 2026-02-20T21:04:28.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:04:28.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:04:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:04:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:04:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:04:27 2026 2026-02-20T21:04:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:04:27 2026 2026-02-20T21:04:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:04:27 2026 2026-02-20T21:04:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:04:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:04:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: b25ea358-0490-4428-8a73-d03a18264814 2026-02-20T21:04:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:04:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test1 100 2026-02-20T21:04:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:04:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:04:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:04:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:04:28.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:04:28.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:04:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:04:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5000.07 bytes/sec: 20 MiB/s 2026-02-20T21:04:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirrors cluster1 2026-02-20T21:04:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:04:28.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:28.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:0 2026-02-20T21:04:28.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:04:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:04:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:04:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:04:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:28.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:04:28.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:04:28.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:28.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:1 2026-02-20T21:04:28.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:04:28.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:28.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:04:28.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:04:28.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:28.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:28.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:28.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:04:28.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:28.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:28.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:28.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:04:28.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:28.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:04:28.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:04:28.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:2 2026-02-20T21:04:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:04:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:04:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:2 2026-02-20T21:04:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=2 2026-02-20T21:04:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:28.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:28.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=2 2026-02-20T21:04:28.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:04:28.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:04:28.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:04:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:3 2026-02-20T21:04:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:04:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:04:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:3 2026-02-20T21:04:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=3 2026-02-20T21:04:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=3 2026-02-20T21:04:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:04:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:04:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:04:28.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-20T21:04:28.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:28.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:28.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:28.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-20T21:04:28.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:28.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:28.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:28.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:04:28.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:04:28.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:28.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-20T21:04:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:04:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:04:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:04:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-20T21:04:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:04:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:04:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:04:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:04:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:04:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:04:29.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:04:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-20T21:04:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:04:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:04:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:04:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:04:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:04:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:29.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:29.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:29.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:29.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-20T21:04:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-20T21:04:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:04:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test1 2026-02-20T21:04:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:04:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:04:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:04:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:04:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:04:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:04:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:04:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:04:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:04:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:04:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:04:29.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:29.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:29.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:29.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test1 2026-02-20T21:04:29.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-20T21:04:29.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:04:29.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test1 2026-02-20T21:04:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:04:29.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:29.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:29.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:04:29.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:04:29.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:29.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:29.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:29.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:04:29.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:29.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:29.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:29.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:04:29.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:04:29.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:04:29.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:04:29.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:29.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:29.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:04:29.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:04:29.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:29.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:29.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:29.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:04:29.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:29.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:29.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:29.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:04:29.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:04:29.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:04:29.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:29.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:29.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:29.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test1 2026-02-20T21:04:29.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-20T21:04:29.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:04:29.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:29.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:29.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test1 2026-02-20T21:04:29.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:04:29.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:04:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:04:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:04:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:04:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:04:29.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:04:29.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:04:29.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:29.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:29.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:04:29.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:04:29.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:29.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:29.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:29.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:04:29.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:29.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:29.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:29.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:04:29.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:04:29.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:04:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test1 2026-02-20T21:04:29.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-20T21:04:29.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:04:29.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:29.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:29.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 1 2026-02-20T21:04:30.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:04:30.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:04:30.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:30.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-20T21:04:30.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:04:30.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:30.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:30.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:30.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:30.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:30.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:30.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:04:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:04:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:04:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:04:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:30.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:30.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:30.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:30.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-20T21:04:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:04:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:04:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:04:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:04:30.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:04:30.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:04:30.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:04:30.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:30.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test1 2026-02-20T21:04:30.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:04:30.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:30.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:04:30.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:04:30.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:30.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:04:30.514 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:04:30.514 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test1 2026-02-20T21:04:30.514 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:30.514 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:30.514 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:30.514 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test1 2026-02-20T21:04:31.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 10 2026-02-20T21:04:31.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test1 2026-02-20T21:04:31.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:04:31.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:31.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:04:31.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:04:31.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:31.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test1.status 2026-02-20T21:04:31.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test1.status 2026-02-20T21:04:31.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:04:31.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test1.status 2026-02-20T21:04:31.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test1.status 2026-02-20T21:04:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test1.status 2026-02-20T21:04:31.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:04:31.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test1.status 2026-02-20T21:04:31.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test1 /tmp/tmp.2FjbyoErqN/cluster2-mirror-test1.status 2026-02-20T21:04:31.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:31.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:31.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:31.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test1.status 2026-02-20T21:04:31.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test1 --format xml 2026-02-20T21:04:31.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:04:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:04:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=10 2026-02-20T21:04:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:04:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:04:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:04:31.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test1 /tmp/tmp.2FjbyoErqN/cluster1-mirror-test1.status 2026-02-20T21:04:31.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:31.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:31.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:31.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test1.status 2026-02-20T21:04:31.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test1 --format xml 2026-02-20T21:04:31.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:04:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:04:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=9 2026-02-20T21:04:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 10 = 9 2026-02-20T21:04:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:04:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.4 2026-02-20T21:04:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test1 /tmp/tmp.2FjbyoErqN/cluster1-mirror-test1.status 2026-02-20T21:04:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test1.status 2026-02-20T21:04:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test1 --format xml 2026-02-20T21:04:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:04:31.757 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:04:31.758 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=9 2026-02-20T21:04:31.758 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 10 = 9 2026-02-20T21:04:31.758 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:04:31.758 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.8 2026-02-20T21:04:32.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test1 /tmp/tmp.2FjbyoErqN/cluster1-mirror-test1.status 2026-02-20T21:04:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test1.status 2026-02-20T21:04:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test1 --format xml 2026-02-20T21:04:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:04:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:04:32.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=10 2026-02-20T21:04:32.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 10 = 10 2026-02-20T21:04:32.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:32.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test1 2026-02-20T21:04:32.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:32.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:32.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:32.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:04:32.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-02-20T21:04:32.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:32.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:32.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:32.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:32.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:32.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:32.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:32.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:33.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-20T21:04:33.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:33.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:33.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:33.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:33.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:33.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:33.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:33.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:33.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-20T21:04:33.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test1: 2026-02-20T21:04:33.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: b25ea358-0490-4428-8a73-d03a18264814 2026-02-20T21:04:33.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:04:33.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621468,"replay_state":"idle"} 2026-02-20T21:04:33.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:04:33.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:04:30' 2026-02-20T21:04:33.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:04:33.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:04:33.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:04:33.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621468,"replay_state":"idle"} 2026-02-20T21:04:33.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:04:33.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:04:33.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:04:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:04:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:04:33.632 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:04:33.632 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:04:33.632 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:04:33.634 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:30' 2026-02-20T21:04:33.634 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-20 21:04:30' 2026-02-20T21:04:33.634 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:33.634 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:33.634 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:33.635 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:33.635 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:04:30' 2026-02-20T21:04:33.635 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:33.635 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:33.635 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:04:33.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtestc5fdae76-300b-4ccc-baee-3e7deda3028dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621462,"remote_snapshot_timestamp":1771621462,"replay_state":"idle"}44114415mirror.1trial1992026-02-20 21:04:32test1b25ea358-0490-4428-8a73-d03a18264814up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621468,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:04:30' 2026-02-20T21:04:33.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:04:33.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-20T21:04:33.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:30' 2026-02-20T21:04:33.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-20T21:04:33.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:04:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:04:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:04:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:04:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -z '' ']' 2026-02-20T21:04:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-20T21:04:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:33.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=down+unknown 2026-02-20T21:04:33.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:33.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:33.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:33.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-20T21:04:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:34.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=down+unknown 2026-02-20T21:04:34.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:34.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:34.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:34.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:34.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-20T21:04:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test1: 2026-02-20T21:04:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: b25ea358-0490-4428-8a73-d03a18264814 2026-02-20T21:04:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: down+unknown 2026-02-20T21:04:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: status not found 2026-02-20T21:04:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20T21:04:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:04:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:04:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:04:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621468,"replay_state":"idle"} 2026-02-20T21:04:34.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:04:30 2026-02-20T21:04:34.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:04:34.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 9 .mirror.primary.b25ea358-0490-4428-8a73-d03a18264814.6ee1fcaa-5242-4fba-8295-8c6fd5a1b1c8 (peer_uuids:[]) 2026-02-20T21:04:34.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 10 .mirror.primary.b25ea358-0490-4428-8a73-d03a18264814.2aa38a13-a475-4520-89ba-f9e49490f08b (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:04:34.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*down+unknown' 2026-02-20T21:04:34.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: down+unknown 2026-02-20T21:04:34.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:04:34.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: status not found 2026-02-20T21:04:34.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:04:34.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo down+unknown 2026-02-20T21:04:34.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:04:34.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: ' 2026-02-20T21:04:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:04:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:04:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update= 2026-02-20T21:04:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-20T21:04:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=down+unknown 2026-02-20T21:04:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local prev_last_update= 2026-02-20T21:04:34.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:34.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:34.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:04:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='UNKNOWNUNKNOWNOK2testc5fdae76-300b-4ccc-baee-3e7deda3028ddown+unknownstatus not foundcluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621462,"remote_snapshot_timestamp":1771621462,"replay_state":"idle"}2026-02-20 21:04:32test1b25ea358-0490-4428-8a73-d03a18264814down+unknownstatus not foundcluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621468,"replay_state":"idle"}2026-02-20 21:04:30' 2026-02-20T21:04:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:04:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-20T21:04:35.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update= 2026-02-20T21:04:35.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-20T21:04:35.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=down+unknown 2026-02-20T21:04:35.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo down+unknown 2026-02-20T21:04:35.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep down+unknown 2026-02-20T21:04:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:down+unknown 2026-02-20T21:04:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror mirror test1 2026-02-20T21:04:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:04:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:04:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:04:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:04:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:35.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-20T21:04:35.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test1.export 2026-02-20T21:04:35.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:04:35.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test1.export 2026-02-20T21:04:35.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-20T21:04:35.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test1.export 2026-02-20T21:04:35.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:04:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test1.export 2026-02-20T21:04:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test1.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test1.export 2026-02-20T21:04:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.2FjbyoErqN/cluster2-mirror-test1.export 2026-02-20T21:04:35.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:04:35.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.2FjbyoErqN/cluster1-mirror-test1.export 2026-02-20T21:04:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:04:35.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror-test1.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test1.export 2026-02-20T21:04:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test1.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test1.export 2026-02-20T21:04:35.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:35.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: test the first image is replaying after restart' 2026-02-20T21:04:35.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:04:35.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:04:35.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:04:35 TEST: test the first image is replaying after restart 2026-02-20T21:04:35.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:04:35 TEST: test the first image is replaying after restart 2026-02-20T21:04:35.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test 100 2026-02-20T21:04:35.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:35.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:35.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:35.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:04:35.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:04:35.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:04:35.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:04:35.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:04:35.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:04:35.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:04:35.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5000.07 bytes/sec: 20 MiB/s 2026-02-20T21:04:35.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:04:35.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:35.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:35.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:35.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:04:35.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:35.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:35.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:35.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:04:35.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:04:35.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:35.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:36.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:04:36.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:36.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:36.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:36.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:04:36.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:04:36.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:04:36.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:04:36.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:04:36.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:04:36.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:36.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:36.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:04:36.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:04:36.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:36.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:04:36.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:04:36.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:04:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:04:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:36.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:36.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:36.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:36.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:04:36.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:no valid command found; 1 closest matches: 2026-02-20T21:04:36.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status mirror/test1 2026-02-20T21:04:36.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:36.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:36.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:04:36.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:04:36.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:36.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:36.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:04:36.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:04:36.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:36.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:36.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:36.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:36.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:36.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:36.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:36.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:04:36.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:36.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:04:36.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:04:36.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:36.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:36.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:04:36.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:04:36.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:36.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:36.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:36.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:36.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:36.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:36.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:36.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:04:36.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:04:36.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:04:36.549 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:36.549 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:36.549 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:36.549 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:04:36.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:04:36.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:04:36.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:04:36.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:04:36.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:04:36.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:04:36.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:04:36.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:36.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:04:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:04:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:04:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:04:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:04:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:04:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:04:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:04:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 11 2026-02-20T21:04:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:04:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:04:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:04:38.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:04:38.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:38.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:04:38.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:04:38.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:04:38.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:04:38.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:04:38.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:04:38.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:04:38.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:04:38.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:04:38.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:38.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:38.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:38.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:04:38.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:04:38.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:04:38.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:04:38.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=11 2026-02-20T21:04:38.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:04:38.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:04:38.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:04:38.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:04:38.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:38.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:38.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:04:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:04:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:04:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:04:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=11 2026-02-20T21:04:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 11 = 11 2026-02-20T21:04:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-20T21:04:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:04:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:04:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:39.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:04:39.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:39.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:39.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:39.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:39.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:39.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:39.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:39.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:39.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:04:39.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:04:39.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: c5fdae76-300b-4ccc-baee-3e7deda3028d 2026-02-20T21:04:39.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:04:39.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621462,"remote_snapshot_timestamp":1771621462,"replay_state":"idle"} 2026-02-20T21:04:39.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:04:39.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:04:32' 2026-02-20T21:04:39.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:04:39.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:04:39.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:04:39.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621462,"remote_snapshot_timestamp":1771621462,"replay_state":"idle"} 2026-02-20T21:04:39.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:04:39.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:04:39.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:04:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:04:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:04:39.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:04:39.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:04:39.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:04:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:32' 2026-02-20T21:04:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:04:32' 2026-02-20T21:04:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:04:32' 2026-02-20T21:04:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:04:39.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtestc5fdae76-300b-4ccc-baee-3e7deda3028dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621462,"remote_snapshot_timestamp":1771621462,"replay_state":"idle"}44114415mirror.1trial1992026-02-20 21:04:32test1b25ea358-0490-4428-8a73-d03a18264814up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621468,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:04:30' 2026-02-20T21:04:39.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:04:39.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:04:39.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:32' 2026-02-20T21:04:39.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:04:39.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:04:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:04:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:04:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:04:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror mirror test 2026-02-20T21:04:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:04:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:04:39.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:04:39.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:04:39.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:04:39.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:39.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.export 2026-02-20T21:04:39.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.export 2026-02-20T21:04:39.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:04:39.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export 2026-02-20T21:04:39.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.export 2026-02-20T21:04:39.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.export 2026-02-20T21:04:39.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:04:39.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:04:39.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:04:39.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export 2026-02-20T21:04:39.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:04:39.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:04:39.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:04:39.603 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:04:39.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:04:39.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:39.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -z '' ']' 2026-02-20T21:04:39.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: stop/start/restart mirror via admin socket' 2026-02-20T21:04:39.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:04:39.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:04:39.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:04:39 TEST: stop/start/restart mirror via admin socket 2026-02-20T21:04:39.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:04:39 TEST: stop/start/restart mirror via admin socket 2026-02-20T21:04:39.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ all_admin_daemons cluster1 rbd mirror stop 2026-02-20T21:04:39.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:39.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:39.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:04:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:0 rbd mirror stop 2026-02-20T21:04:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:04:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:04:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:04:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:04:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:04:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:39.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:39.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:39.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:39.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:39.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:39.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:39.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:39.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:39.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:39.700 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:39.700 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:39.700 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:39.701 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop 2026-02-20T21:04:39.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:39.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:1 rbd mirror stop 2026-02-20T21:04:39.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:04:39.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:39.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:39.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:04:39.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:04:39.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:39.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:39.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:39.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:04:39.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:39.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:39.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:39.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:04:39.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:39.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:04:39.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:04:39.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:39.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:39.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:04:39.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:04:39.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:39.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:39.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:39.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:39.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:39.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:39.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:39.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:04:39.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:39.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:04:39.758 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:39.759 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:39.759 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:39.759 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror stop 2026-02-20T21:04:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:2 rbd mirror stop 2026-02-20T21:04:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:04:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:04:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:2 2026-02-20T21:04:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=2 2026-02-20T21:04:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=2 2026-02-20T21:04:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:04:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:04:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:04:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:04:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:04:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:04:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:04:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:04:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:04:39.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:39.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:39.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:39.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror stop 2026-02-20T21:04:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:3 rbd mirror stop 2026-02-20T21:04:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:04:39.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:04:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:3 2026-02-20T21:04:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=3 2026-02-20T21:04:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=3 2026-02-20T21:04:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:04:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:04:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:04:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:04:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:04:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:04:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:39.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:04:39.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:04:39.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:04:39.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:39.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:39.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:39.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror stop 2026-02-20T21:04:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:04:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:04:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:04:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:04:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:40.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:04:40.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:40.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:40.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:40.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:04:40.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:04:40.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:04:40.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:04:40.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:04:40.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:04:40.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:40.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:40.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:04:40.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:04:40.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:40.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:04:40.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:04:40.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:04:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:04:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:04:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:04:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:04:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:04:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:04:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:04:40.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:no valid command found; 1 closest matches: 2026-02-20T21:04:40.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status mirror/test1 2026-02-20T21:04:40.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:40.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:40.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:04:40.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:04:40.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:40.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:40.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:04:40.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:04:40.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:40.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:40.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:40.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:40.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:40.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:40.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:40.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:04:40.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:40.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:04:40.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:04:40.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:40.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:40.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:04:40.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:04:40.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:40.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:40.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:40.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:40.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:40.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:40.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:04:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:04:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:04:40.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:40.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:40.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:40.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:04:41.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:04:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:04:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:04:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:04:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:04:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test1 2026-02-20T21:04:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test1 stopped 2026-02-20T21:04:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:04:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:04:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:42.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test1 stopped 2026-02-20T21:04:42.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:42.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:42.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:42.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:04:42.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:04:42.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:04:42.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-20T21:04:42.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:04:42.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:42.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:42.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:04:42.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:04:42.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:04:42.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:42.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:04:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:04:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:04:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-20T21:04:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:04:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:42.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:42.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:42.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:42.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:42.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:42.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:42.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:42.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:42.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:04:42.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:42.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:42.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:42.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:04:42.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:42.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:42.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:42.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:42.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:42.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:42.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:42.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:04:42.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:04:42.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:42.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:42.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:42.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:42.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-20T21:04:42.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:04:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:04:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:04:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:04:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:04:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:04:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:04:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:04:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:43.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:04:43.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:43.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:43.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:43.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:43.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:04:43.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:04:43.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: c5fdae76-300b-4ccc-baee-3e7deda3028d 2026-02-20T21:04:43.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:04:43.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: stopped 2026-02-20T21:04:43.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:04:43.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:04:39' 2026-02-20T21:04:43.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:04:43.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:04:43.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:04:43.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: stopped 2026-02-20T21:04:43.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:04:43.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:04:43.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:04:43.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:04:43.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:04:43.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:04:43.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:04:43.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:04:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:39' 2026-02-20T21:04:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:04:39' 2026-02-20T21:04:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:04:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:04:39' 2026-02-20T21:04:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:43.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:04:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtestc5fdae76-300b-4ccc-baee-3e7deda3028dup+stoppedstopped44114415mirror.1trial1992026-02-20 21:04:39test1b25ea358-0490-4428-8a73-d03a18264814up+stoppedstopped44044409mirror.0trial1992026-02-20 21:04:39' 2026-02-20T21:04:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:04:43.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:04:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:39' 2026-02-20T21:04:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:04:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:04:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:04:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:04:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:04:43.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:43.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:43.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+stopped 2026-02-20T21:04:43.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:43.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:43.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:43.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:04:43.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:43.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:43.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:43.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:44.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+stopped '' '' 2026-02-20T21:04:44.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:44.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:44.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:44.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:04:44.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:44.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:44.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:44.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:44.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-20T21:04:44.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test1: 2026-02-20T21:04:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: b25ea358-0490-4428-8a73-d03a18264814 2026-02-20T21:04:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:04:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: stopped 2026-02-20T21:04:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:04:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:04:39' 2026-02-20T21:04:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:04:44.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:04:44.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:04:44.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: stopped 2026-02-20T21:04:44.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:04:44.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:04:44.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:04:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:04:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:04:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:04:44.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:04:44.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:04:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:39' 2026-02-20T21:04:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+stopped '2026-02-20 21:04:39' 2026-02-20T21:04:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:04:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:04:39' 2026-02-20T21:04:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:04:44.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtestc5fdae76-300b-4ccc-baee-3e7deda3028dup+stoppedstopped44114415mirror.1trial1992026-02-20 21:04:39test1b25ea358-0490-4428-8a73-d03a18264814up+stoppedstopped44044409mirror.0trial1992026-02-20 21:04:39' 2026-02-20T21:04:44.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:04:44.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-20T21:04:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:39' 2026-02-20T21:04:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-20T21:04:44.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:04:44.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:04:44.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:04:44.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:04:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ all_admin_daemons cluster1 rbd mirror start 2026-02-20T21:04:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:04:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:0 rbd mirror start 2026-02-20T21:04:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:04:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:04:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:04:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:04:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:04:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:44.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:44.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:44.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:44.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:44.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:44.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:44.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start 2026-02-20T21:04:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:1 rbd mirror start 2026-02-20T21:04:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:04:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:04:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror start 2026-02-20T21:04:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:2 rbd mirror start 2026-02-20T21:04:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:04:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:04:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:2 2026-02-20T21:04:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=2 2026-02-20T21:04:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=2 2026-02-20T21:04:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:04:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:04:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:04:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:04:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:04:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:04:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:04:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:04:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:04:44.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:44.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:44.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:44.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror start 2026-02-20T21:04:44.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:3 rbd mirror start 2026-02-20T21:04:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:04:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:04:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:3 2026-02-20T21:04:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=3 2026-02-20T21:04:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=3 2026-02-20T21:04:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:04:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:04:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:04:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:04:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:04:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:44.433 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:44.433 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:44.433 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:04:44.433 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:44.433 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:44.433 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:44.433 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:04:44.433 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:04:44.433 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:04:44.440 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:44.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:44.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:44.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror start 2026-02-20T21:04:44.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:04:44.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:44.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:44.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:44.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:04:44.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:44.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:44.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:44.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:04:44.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:04:44.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:44.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:04:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:04:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:04:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:04:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:04:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:04:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:04:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:04:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:04:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:04:45.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:04:45.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:04:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:04:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:04:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:45.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:45.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:45.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:04:45.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:45.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:45.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:45.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:04:45.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:45.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:45.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:45.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:45.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:45.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:45.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:45.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:04:45.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:04:45.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:45.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:45.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:45.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:45.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:04:45.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:no valid command found; 1 closest matches: 2026-02-20T21:04:45.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status mirror/test1 2026-02-20T21:04:45.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:45.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:45.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:04:45.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:04:45.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:45.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:45.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:04:45.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:04:45.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:45.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:45.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:45.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:45.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:45.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:45.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:45.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:04:45.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:45.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:04:45.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:04:45.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:45.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:45.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:04:45.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:04:45.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:45.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:45.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:45.549 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:45.549 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:45.549 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:45.549 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:45.549 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:04:45.549 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:04:45.549 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:04:45.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:45.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:45.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:45.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:04:45.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:04:45.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:04:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:04:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:04:45.607 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:04:45.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:04:45.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:04:45.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-20T21:04:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-20T21:04:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:04:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:04:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-20T21:04:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:04:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:04:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:04:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-20T21:04:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:04:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:04:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:04:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:04:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:04:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:04:46.614 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:04:46.614 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:46.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-20T21:04:46.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:04:46.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:46.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:46.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:46.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:46.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:46.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:46.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:46.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:46.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:04:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:04:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:46.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:46.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:46.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:46.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:46.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:46.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:04:46.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:04:46.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:46.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:46.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:46.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:46.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-20T21:04:46.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:04:46.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:04:46.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:04:46.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:04:46.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:04:46.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:04:46.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:04:46.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:46.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:04:46.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:46.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:46.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:46.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:46.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:46.672 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:46.672 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:46.672 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:47.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:04:47.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:47.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:47.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:47.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:47.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:47.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:47.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:47.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:47.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:04:47.702 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:04:47.702 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: c5fdae76-300b-4ccc-baee-3e7deda3028d 2026-02-20T21:04:47.702 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:04:47.702 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"} 2026-02-20T21:04:47.702 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:04:47.702 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:04:44' 2026-02-20T21:04:47.702 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:04:47.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:04:47.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:04:47.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"} 2026-02-20T21:04:47.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:04:47.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:04:47.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:04:47.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:04:47.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:04:47.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:04:47.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:04:47.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:04:47.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:44' 2026-02-20T21:04:47.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:04:44' 2026-02-20T21:04:47.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:47.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:47.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:47.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:47.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:04:44' 2026-02-20T21:04:47.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:47.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:47.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:04:47.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtestc5fdae76-300b-4ccc-baee-3e7deda3028dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"}44114415mirror.1trial1992026-02-20 21:04:44test1b25ea358-0490-4428-8a73-d03a18264814up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621471,"remote_snapshot_timestamp":1771621471,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:04:44' 2026-02-20T21:04:47.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:04:47.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:04:47.744 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:44' 2026-02-20T21:04:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:04:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:04:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:04:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:04:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:04:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-02-20T21:04:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:47.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-20T21:04:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-20T21:04:48.780 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test1: 2026-02-20T21:04:48.780 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: b25ea358-0490-4428-8a73-d03a18264814 2026-02-20T21:04:48.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:04:48.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621471,"remote_snapshot_timestamp":1771621471,"replay_state":"idle"} 2026-02-20T21:04:48.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:04:48.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:04:44' 2026-02-20T21:04:48.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:04:48.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:04:48.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:04:48.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621471,"remote_snapshot_timestamp":1771621471,"replay_state":"idle"} 2026-02-20T21:04:48.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:04:48.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:04:48.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:04:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:04:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:04:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:04:48.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:04:48.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:04:48.786 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:44' 2026-02-20T21:04:48.786 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-20 21:04:44' 2026-02-20T21:04:48.787 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:48.787 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:48.787 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:48.787 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:48.787 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:04:44' 2026-02-20T21:04:48.787 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:48.787 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:48.787 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:04:48.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtestc5fdae76-300b-4ccc-baee-3e7deda3028dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"}44114415mirror.1trial1992026-02-20 21:04:44test1b25ea358-0490-4428-8a73-d03a18264814up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621471,"remote_snapshot_timestamp":1771621471,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:04:44' 2026-02-20T21:04:48.821 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:04:48.821 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-20T21:04:48.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:44' 2026-02-20T21:04:48.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-20T21:04:48.824 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:04:48.824 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:04:48.824 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:04:48.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:04:48.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:48.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:48.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ all_admin_daemons cluster1 rbd mirror restart 2026-02-20T21:04:48.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:48.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:48.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:04:48.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:48.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:0 rbd mirror restart 2026-02-20T21:04:48.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:04:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:04:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:04:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:04:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:04:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:48.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:48.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:48.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:48.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:48.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:48.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:48.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:48.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart 2026-02-20T21:04:48.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:1 rbd mirror restart 2026-02-20T21:04:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:04:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:04:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:04:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:48.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:04:48.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:48.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:48.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:48.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:04:48.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:48.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:04:48.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:04:48.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:48.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:48.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:04:48.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:04:48.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:48.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:48.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:48.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:48.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:48.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:48.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:48.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:04:48.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:48.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:04:48.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:48.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:48.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:48.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart 2026-02-20T21:04:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:2 rbd mirror restart 2026-02-20T21:04:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:04:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:04:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:2 2026-02-20T21:04:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=2 2026-02-20T21:04:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=2 2026-02-20T21:04:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:04:48.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:04:48.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:04:48.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:48.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:48.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:04:48.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:04:48.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:48.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:48.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:48.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:04:48.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:48.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:48.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:48.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:04:48.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:04:48.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:04:48.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:48.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:48.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:48.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror restart 2026-02-20T21:04:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:3 rbd mirror restart 2026-02-20T21:04:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:04:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:04:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:3 2026-02-20T21:04:49.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=3 2026-02-20T21:04:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=3 2026-02-20T21:04:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:04:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:04:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:04:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:49.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:04:49.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:04:49.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:49.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:49.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:49.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:04:49.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:49.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:49.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:49.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:04:49.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:04:49.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:04:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror restart 2026-02-20T21:04:49.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:04:49.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:49.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:49.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:04:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:04:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:04:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:04:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:04:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:04:50.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:04:50.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:04:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:04:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:04:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:04:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:04:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:04:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:04:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:04:50.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:04:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:04:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:04:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:04:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:04:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:04:50.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:50.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:50.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:50.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:50.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:04:50.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:no valid command found; 1 closest matches: 2026-02-20T21:04:50.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status mirror/test1 2026-02-20T21:04:50.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:50.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:50.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:04:50.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:04:50.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:50.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:50.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:04:50.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:04:50.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:50.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:04:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:04:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:04:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:04:50.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:04:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:04:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:04:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:04:50.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:50.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:50.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:50.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:04:50.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:04:50.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:04:50.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:04:50.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:04:50.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:04:50.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:04:50.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:04:50.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-20T21:04:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-20T21:04:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:04:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:04:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:51.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-20T21:04:51.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:51.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:51.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:51.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:04:51.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:04:51.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:04:51.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-20T21:04:51.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:04:51.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:04:51.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:51.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:51.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:04:51.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:04:51.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:51.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:04:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:04:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:04:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-20T21:04:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:04:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:04:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:04:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:51.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:51.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:51.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:51.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:04:51.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:04:51.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-20T21:04:51.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:04:51.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:04:51.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:04:51.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:04:51.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:04:51.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:04:51.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:04:51.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:51.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:04:51.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:51.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:51.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:51.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:51.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:51.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:51.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:51.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:04:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:52.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:04:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:04:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: c5fdae76-300b-4ccc-baee-3e7deda3028d 2026-02-20T21:04:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:04:52.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"} 2026-02-20T21:04:52.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:04:52.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:04:48' 2026-02-20T21:04:52.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:04:52.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:04:52.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:04:52.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"} 2026-02-20T21:04:52.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:04:52.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:04:52.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:04:52.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:04:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:04:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:04:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:04:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:04:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:48' 2026-02-20T21:04:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:04:48' 2026-02-20T21:04:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:52.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:52.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:52.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:04:48' 2026-02-20T21:04:52.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:52.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:52.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:04:52.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtestc5fdae76-300b-4ccc-baee-3e7deda3028dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"}44114415mirror.1trial1992026-02-20 21:04:48test1b25ea358-0490-4428-8a73-d03a18264814up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621471,"remote_snapshot_timestamp":1771621471,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:04:48' 2026-02-20T21:04:52.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:04:52.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:04:52.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:48' 2026-02-20T21:04:52.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:04:52.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:04:52.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:04:52.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:04:52.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:04:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-02-20T21:04:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:53.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-20T21:04:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-20T21:04:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test1: 2026-02-20T21:04:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: b25ea358-0490-4428-8a73-d03a18264814 2026-02-20T21:04:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:04:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621471,"remote_snapshot_timestamp":1771621471,"replay_state":"idle"} 2026-02-20T21:04:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:04:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:04:48' 2026-02-20T21:04:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:04:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:04:53.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:04:53.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621471,"remote_snapshot_timestamp":1771621471,"replay_state":"idle"} 2026-02-20T21:04:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:04:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:04:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:04:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:04:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:04:53.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:04:53.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:04:53.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:04:53.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:48' 2026-02-20T21:04:53.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-20 21:04:48' 2026-02-20T21:04:53.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:53.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:53.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:53.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:04:53.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:04:48' 2026-02-20T21:04:53.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:53.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:53.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:04:53.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtestc5fdae76-300b-4ccc-baee-3e7deda3028dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"}44114415mirror.1trial1992026-02-20 21:04:48test1b25ea358-0490-4428-8a73-d03a18264814up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621471,"remote_snapshot_timestamp":1771621471,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:04:48' 2026-02-20T21:04:53.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:04:53.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-20T21:04:53.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:48' 2026-02-20T21:04:53.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-20T21:04:53.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:04:53.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:04:53.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:04:53.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:04:53.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:53.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:53.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ all_admin_daemons cluster1 rbd mirror stop 2026-02-20T21:04:53.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:53.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:04:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:0 rbd mirror stop 2026-02-20T21:04:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:04:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:04:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:04:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:04:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:04:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:53.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:53.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:53.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:53.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:53.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:53.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:53.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:53.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:53.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:53.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:53.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:53.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:53.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:53.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:53.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:53.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop 2026-02-20T21:04:53.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:53.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:1 rbd mirror stop 2026-02-20T21:04:53.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:04:53.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:53.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:53.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:04:53.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:04:53.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:53.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:53.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:53.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:04:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:04:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:04:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:04:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:04:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:04:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:04:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:04:53.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:53.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:53.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:53.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror stop 2026-02-20T21:04:53.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:53.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:2 rbd mirror stop 2026-02-20T21:04:53.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:04:53.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:53.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:53.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:04:53.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:2 2026-02-20T21:04:53.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:53.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:53.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:53.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=2 2026-02-20T21:04:53.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:53.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:53.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:53.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=2 2026-02-20T21:04:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:04:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:04:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:04:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:04:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:04:53.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:53.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:53.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:53.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:04:53.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:53.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:53.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:53.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:04:53.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:04:53.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:04:53.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:53.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:53.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:53.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror stop 2026-02-20T21:04:53.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:53.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:3 rbd mirror stop 2026-02-20T21:04:53.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:04:53.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:53.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:53.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:04:53.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:3 2026-02-20T21:04:53.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:53.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:53.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:53.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=3 2026-02-20T21:04:53.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:53.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:53.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:53.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=3 2026-02-20T21:04:53.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:04:53.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:04:53.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:04:53.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:53.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:53.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:04:53.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:04:53.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:53.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:53.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:53.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:04:53.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:53.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:53.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:53.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:04:53.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:04:53.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:04:53.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:53.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:53.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:53.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror stop 2026-02-20T21:04:53.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:04:53.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:53.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:53.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:53.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:04:53.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:53.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:53.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:53.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:04:53.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:04:53.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:53.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:54.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:04:54.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:54.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:54.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:54.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:04:54.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:04:54.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:04:54.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:04:54.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:04:54.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:54.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:04:54.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:54.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:04:54.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:04:54.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:54.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:04:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:04:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:04:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:04:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:04:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:04:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:04:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:54.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:04:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:04:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:04:54.681 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:no valid command found; 1 closest matches: 2026-02-20T21:04:54.681 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status mirror/test1 2026-02-20T21:04:54.681 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:54.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:54.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:04:54.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:04:54.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:04:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:04:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:04:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:04:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:04:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:04:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:04:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:04:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:04:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:04:54.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:54.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:54.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:54.695 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:04:54.737 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:04:54.738 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:04:54.738 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:04:54.738 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:04:54.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:04:54.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:54.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test1 2026-02-20T21:04:54.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:54.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:54.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:54.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test1 stopped 2026-02-20T21:04:54.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:54.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:54.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:54.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:04:54.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:04:54.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:54.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:55.742 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test1 stopped 2026-02-20T21:04:55.742 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:55.742 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:55.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:55.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:04:55.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:04:55.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:04:55.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-20T21:04:55.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:04:55.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:55.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:55.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:04:55.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:04:55.744 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:55.744 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:04:55.744 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:04:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:04:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:04:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-20T21:04:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:04:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:04:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:55.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:55.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:55.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:04:55.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:55.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:55.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:55.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:55.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:55.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:55.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:55.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:04:55.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:04:55.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:55.757 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:55.757 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:55.758 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:55.758 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-20T21:04:55.801 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:04:55.802 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:04:55.802 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:04:55.802 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:04:55.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:04:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:04:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:04:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:56.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:04:56.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:56.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:56.805 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:56.805 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:04:56.805 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:56.805 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:56.805 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:56.805 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:56.805 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:04:56.836 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:04:56.836 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: c5fdae76-300b-4ccc-baee-3e7deda3028d 2026-02-20T21:04:56.836 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:04:56.836 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: stopped 2026-02-20T21:04:56.836 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:04:56.836 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:04:53' 2026-02-20T21:04:56.836 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:04:56.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:04:56.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:04:56.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: stopped 2026-02-20T21:04:56.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:04:56.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:04:56.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:04:56.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:04:56.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:04:56.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:04:56.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:04:56.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:04:56.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:53' 2026-02-20T21:04:56.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:04:53' 2026-02-20T21:04:56.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:56.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:56.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:56.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:04:56.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:04:53' 2026-02-20T21:04:56.843 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:56.843 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:56.843 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:04:56.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtestc5fdae76-300b-4ccc-baee-3e7deda3028dup+stoppedstopped44114415mirror.1trial1992026-02-20 21:04:53test1b25ea358-0490-4428-8a73-d03a18264814up+stoppedstopped44044409mirror.0trial1992026-02-20 21:04:53' 2026-02-20T21:04:56.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:04:56.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:04:56.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:53' 2026-02-20T21:04:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:04:56.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:04:56.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:04:56.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:04:56.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:04:56.882 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:56.882 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:56.882 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+stopped 2026-02-20T21:04:56.882 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:56.882 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:56.882 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:56.882 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:04:56.882 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:56.882 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:56.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:56.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:57.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+stopped '' '' 2026-02-20T21:04:57.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:57.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:57.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:57.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:04:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:04:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:04:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-20T21:04:57.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test1: 2026-02-20T21:04:57.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: b25ea358-0490-4428-8a73-d03a18264814 2026-02-20T21:04:57.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:04:57.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: stopped 2026-02-20T21:04:57.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:04:57.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:04:53' 2026-02-20T21:04:57.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:04:57.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:04:57.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:04:57.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: stopped 2026-02-20T21:04:57.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:04:57.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:04:57.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:04:57.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:04:57.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:04:57.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:04:57.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:04:57.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:04:57.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:53' 2026-02-20T21:04:57.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+stopped '2026-02-20 21:04:53' 2026-02-20T21:04:57.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:57.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:57.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:57.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:04:57.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:04:53' 2026-02-20T21:04:57.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:04:57.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:04:57.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:04:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtestc5fdae76-300b-4ccc-baee-3e7deda3028dup+stoppedstopped44114415mirror.1trial1992026-02-20 21:04:53test1b25ea358-0490-4428-8a73-d03a18264814up+stoppedstopped44044409mirror.0trial1992026-02-20 21:04:53' 2026-02-20T21:04:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:04:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-20T21:04:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:53' 2026-02-20T21:04:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-20T21:04:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:04:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:04:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:04:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:04:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:57.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:57.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ all_admin_daemons cluster1 rbd mirror restart 2026-02-20T21:04:57.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:57.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:57.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:04:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:0 rbd mirror restart 2026-02-20T21:04:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:04:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:04:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:04:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:04:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:04:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:57.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart 2026-02-20T21:04:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:1 rbd mirror restart 2026-02-20T21:04:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:04:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:58.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:04:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:04:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:04:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:04:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:04:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:04:58.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:58.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:58.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:04:58.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:04:58.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:58.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:58.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:58.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:58.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:58.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:58.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:58.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:04:58.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:58.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:04:58.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:58.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:58.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:58.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart 2026-02-20T21:04:58.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:58.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:2 rbd mirror restart 2026-02-20T21:04:58.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:04:58.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:58.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:04:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:2 2026-02-20T21:04:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=2 2026-02-20T21:04:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=2 2026-02-20T21:04:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:04:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:04:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:04:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:04:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:04:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:04:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:04:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:58.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:58.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:04:58.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:04:58.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:04:58.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:04:58.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror restart 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:3 rbd mirror restart 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:3 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=3 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=3 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:04:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:04:58.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:04:58.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:04:58.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:04:58.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:04:58.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:58.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:58.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:04:58.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror restart 2026-02-20T21:04:58.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:04:58.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:58.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:58.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:58.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:04:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:04:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:04:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:04:59.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:04:59.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:59.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:59.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:04:59.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:04:59.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:04:59.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:04:59.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:04:59.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:04:59.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:04:59.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:59.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:04:59.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:04:59.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:04:59.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:59.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:04:59.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:04:59.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:04:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:59.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:04:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:04:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:04:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:04:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:04:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:04:59.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:04:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:04:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:04:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:04:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:04:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:04:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:04:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:04:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:04:59.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:59.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:59.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:04:59.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:04:59.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:no valid command found; 1 closest matches: 2026-02-20T21:04:59.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status mirror/test1 2026-02-20T21:04:59.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:04:59.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:04:59.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:04:59.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:04:59.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:04:59.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:04:59.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:04:59.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:04:59.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:04:59.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:04:59.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:04:59.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:04:59.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:59.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:04:59.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:59.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:04:59.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:04:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:04:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:04:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:04:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:04:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:04:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:04:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:04:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:04:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:04:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:04:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:04:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:04:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:04:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:04:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:04:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:04:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:04:59.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:04:59.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:04:59.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:04:59.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:04:59.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:04:59.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:04:59.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:04:59.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:04:59.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-20T21:04:59.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:59.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:59.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:59.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-20T21:04:59.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:04:59.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:04:59.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:04:59.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:04:59.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:04:59.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:04:59.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-20T21:05:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:05:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:05:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:05:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-20T21:05:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:05:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:05:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:05:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:05:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:05:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:00.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:05:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-20T21:05:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:05:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:05:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:05:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:05:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:05:00.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:00.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:00.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:00.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:00.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-20T21:05:00.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:05:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:05:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:05:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:05:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:05:00.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:05:00.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:05:00.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:00.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:05:00.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:00.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:00.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:00.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:00.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:00.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:00.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:00.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:05:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:05:01.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:05:01.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: c5fdae76-300b-4ccc-baee-3e7deda3028d 2026-02-20T21:05:01.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:05:01.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"} 2026-02-20T21:05:01.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:05:01.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:04:58' 2026-02-20T21:05:01.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:05:01.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:05:01.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:05:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"} 2026-02-20T21:05:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:05:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:05:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:05:01.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:05:01.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:05:01.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:05:01.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:05:01.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:05:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:58' 2026-02-20T21:05:01.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:04:58' 2026-02-20T21:05:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:04:58' 2026-02-20T21:05:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:05:01.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtestc5fdae76-300b-4ccc-baee-3e7deda3028dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"}44114415mirror.1trial1992026-02-20 21:04:58test1b25ea358-0490-4428-8a73-d03a18264814up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621471,"remote_snapshot_timestamp":1771621471,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:04:58' 2026-02-20T21:05:01.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:05:01.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:05:01.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:58' 2026-02-20T21:05:01.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:05:01.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:05:01.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:05:01.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:05:01.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:05:01.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:01.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:01.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-02-20T21:05:01.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:01.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:01.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:01.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:01.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:01.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:01.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:01.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:02.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-20T21:05:02.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:02.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:02.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:02.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:02.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:02.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:02.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:02.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:02.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-20T21:05:02.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test1: 2026-02-20T21:05:02.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: b25ea358-0490-4428-8a73-d03a18264814 2026-02-20T21:05:02.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:05:02.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621471,"remote_snapshot_timestamp":1771621471,"replay_state":"idle"} 2026-02-20T21:05:02.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:05:02.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:04:58' 2026-02-20T21:05:02.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:05:02.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:05:02.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:05:02.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621471,"remote_snapshot_timestamp":1771621471,"replay_state":"idle"} 2026-02-20T21:05:02.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:05:02.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:05:02.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:05:02.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:05:02.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:05:02.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:05:02.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:05:02.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:05:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:58' 2026-02-20T21:05:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-20 21:04:58' 2026-02-20T21:05:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:04:58' 2026-02-20T21:05:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:05:02.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtestc5fdae76-300b-4ccc-baee-3e7deda3028dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"}44114415mirror.1trial1992026-02-20 21:04:58test1b25ea358-0490-4428-8a73-d03a18264814up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621471,"remote_snapshot_timestamp":1771621471,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:04:58' 2026-02-20T21:05:02.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:05:02.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-20T21:05:02.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:04:58' 2026-02-20T21:05:02.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-20T21:05:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:05:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:05:02.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:05:02.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:05:02.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:02.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:02.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ all_admin_daemons cluster1 rbd mirror stop mirror cluster2-DNE 2026-02-20T21:05:02.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:02.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:02.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:05:02.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:02.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:0 rbd mirror stop mirror cluster2-DNE 2026-02-20T21:05:02.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:05:02.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:02.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:02.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:02.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:05:02.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:02.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:02.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:02.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:05:02.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:02.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:02.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:02.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:05:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:05:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:02.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:02.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:02.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:02.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop mirror cluster2-DNE 2026-02-20T21:05:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:1 rbd mirror stop mirror cluster2-DNE 2026-02-20T21:05:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:05:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:05:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:05:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:05:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:05:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:05:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:05:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:05:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:05:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:05:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:05:02.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:02.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:02.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:02.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror stop mirror cluster2-DNE 2026-02-20T21:05:02.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:02.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:2 rbd mirror stop mirror cluster2-DNE 2026-02-20T21:05:02.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:05:02.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:02.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:05:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:2 2026-02-20T21:05:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=2 2026-02-20T21:05:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:05:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=2 2026-02-20T21:05:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:05:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:05:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:05:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:05:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:05:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:02.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:05:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:05:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:05:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:05:02.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:05:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:02.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror stop mirror cluster2-DNE 2026-02-20T21:05:02.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:02.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:3 rbd mirror stop mirror cluster2-DNE 2026-02-20T21:05:02.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:05:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:05:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:3 2026-02-20T21:05:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=3 2026-02-20T21:05:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:05:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=3 2026-02-20T21:05:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:05:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:05:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:05:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:05:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:05:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:05:02.713 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:05:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:05:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:05:02.714 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:05:02.721 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:02.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:02.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:02.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror stop mirror cluster2-DNE 2026-02-20T21:05:02.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:05:02.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:02.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:02.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:02.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:05:02.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:02.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:02.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:02.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:05:02.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:05:02.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:02.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:05:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:05:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:05:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:05:03.770 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:05:03.770 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:05:03.770 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:05:03.770 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:03.770 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:03.770 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:05:03.771 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:05:03.771 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:03.771 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:05:03.771 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:03.771 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:05:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:05:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:05:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:03.775 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:03.775 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:03.775 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:03.775 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:03.775 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:03.775 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:03.775 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:03.775 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:03.775 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:03.775 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:05:03.775 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:03.775 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:05:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:05:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:05:03.776 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:03.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:03.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:03.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:03.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:05:03.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:no valid command found; 1 closest matches: 2026-02-20T21:05:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status mirror/test1 2026-02-20T21:05:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:05:03.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:03.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:05:03.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:05:03.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:03.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:03.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:05:03.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:05:03.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:05:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:05:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:05:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:05:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:05:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:05:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:05:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:05:03.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:03.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:03.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:03.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:05:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:05:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:05:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:05:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:05:03.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:05:03.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test1 2026-02-20T21:05:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test1 stopped 2026-02-20T21:05:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:05:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:05:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:03.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:04.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test1 stopped 2026-02-20T21:05:04.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:04.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:04.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:04.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:05:04.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:05:04.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:05:04.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-20T21:05:04.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:05:04.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:05:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:05:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:05:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:05:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:04.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:05:04.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:04.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-20T21:05:04.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:05:04.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:04.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:04.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:04.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:04.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:04.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:04.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:04.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:04.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:04.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:04.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:04.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:04.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:04.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:04.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:05:04.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:04.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:04.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:04.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:05:04.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:04.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:04.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:04.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:04.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:04.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:04.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:04.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:05:04.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:05:04.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:04.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:04.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:04.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-20T21:05:04.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:05:04.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:05:04.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:05:04.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:05:04.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:05:04.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:04.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:05:04.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:04.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:04.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:04.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:05:04.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:04.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:04.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:04.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:05.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:05:05.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:05.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:05.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:05.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:05:05.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:05.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:05.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:05.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:05.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:05:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:05:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: c5fdae76-300b-4ccc-baee-3e7deda3028d 2026-02-20T21:05:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:05:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: stopped 2026-02-20T21:05:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:05:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:05:02' 2026-02-20T21:05:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:05:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:05:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:05:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: stopped 2026-02-20T21:05:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:05:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:05:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:05:05.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:05:05.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:05:05.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:05:05.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:05:05.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:05:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:02' 2026-02-20T21:05:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:05:02' 2026-02-20T21:05:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:05:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:05:02' 2026-02-20T21:05:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:05.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:05:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtestc5fdae76-300b-4ccc-baee-3e7deda3028dup+stoppedstopped44114415mirror.1trial1992026-02-20 21:05:02test1b25ea358-0490-4428-8a73-d03a18264814up+stoppedstopped44044409mirror.0trial1992026-02-20 21:05:02' 2026-02-20T21:05:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:05:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:05:06.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:02' 2026-02-20T21:05:06.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:05:06.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:05:06.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:05:06.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:05:06.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:05:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+stopped 2026-02-20T21:05:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:05:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:06.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:07.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+stopped '' '' 2026-02-20T21:05:07.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:07.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:07.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:07.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:05:07.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:07.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-20T21:05:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test1: 2026-02-20T21:05:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: b25ea358-0490-4428-8a73-d03a18264814 2026-02-20T21:05:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:05:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: stopped 2026-02-20T21:05:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:05:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:05:02' 2026-02-20T21:05:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:05:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:05:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:05:07.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: stopped 2026-02-20T21:05:07.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:05:07.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:05:07.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:05:07.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:05:07.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:05:07.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:05:07.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:05:07.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:05:07.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:02' 2026-02-20T21:05:07.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+stopped '2026-02-20 21:05:02' 2026-02-20T21:05:07.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:07.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:05:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:05:02' 2026-02-20T21:05:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:05:07.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtestc5fdae76-300b-4ccc-baee-3e7deda3028dup+stoppedstopped44114415mirror.1trial1992026-02-20 21:05:02test1b25ea358-0490-4428-8a73-d03a18264814up+stoppedstopped44044409mirror.0trial1992026-02-20 21:05:02' 2026-02-20T21:05:07.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:05:07.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-20T21:05:07.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:02' 2026-02-20T21:05:07.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-20T21:05:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:05:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:05:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:05:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:05:07.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:07.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:07.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemons cluster1 rbd mirror start mirror/test 2026-02-20T21:05:07.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_instance=cluster1 2026-02-20T21:05:07.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:07.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:07.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance=cluster1 2026-02-20T21:05:07.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loop_instance 2026-02-20T21:05:07.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:07.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:05:07.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:07.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:05:07.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:07.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:0 rbd mirror start mirror/test 2026-02-20T21:05:07.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:05:07.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:07.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:07.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:07.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:05:07.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:07.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:07.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:07.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:05:07.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:07.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:07.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:07.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:05:07.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:05:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:07.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:07.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:07.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:07.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start mirror/test 2026-02-20T21:05:07.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:no valid command found; 1 closest matches: 2026-02-20T21:05:07.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror start mirror/test1 2026-02-20T21:05:07.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:05:07.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:07.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:1 rbd mirror start mirror/test 2026-02-20T21:05:07.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:05:07.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:05:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:05:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:05:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:07.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:05:07.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:07.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:05:07.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:05:07.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:07.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:07.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:05:07.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:05:07.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:07.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:07.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:07.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:07.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:07.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:07.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:07.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:05:07.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:07.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:05:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror start mirror/test 2026-02-20T21:05:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:05:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:05:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:05:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:05:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:08.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:05:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:05:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:05:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:05:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:05:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:05:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:05:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:05:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:05:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:05:08.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:08.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:05:08.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:08.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:05:08.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:05:08.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:08.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:05:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:05:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:05:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:05:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:08.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:08.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:08.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:08.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:05:08.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:no valid command found; 1 closest matches: 2026-02-20T21:05:08.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status mirror/test1 2026-02-20T21:05:08.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:05:08.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:05:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:05:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:05:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:05:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:05:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:05:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:05:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:05:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:05:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:05:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:05:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:05:08.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:08.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:08.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:08.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:05:08.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:05:08.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:05:08.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:05:08.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:05:08.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:05:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:05:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:05:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:05:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:05:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:09.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:09.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:09.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:09.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:05:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:05:09.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: c5fdae76-300b-4ccc-baee-3e7deda3028d 2026-02-20T21:05:09.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:05:09.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"} 2026-02-20T21:05:09.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:05:09.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:05:07' 2026-02-20T21:05:09.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:05:09.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:05:09.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:05:09.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"} 2026-02-20T21:05:09.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:05:09.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:05:09.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:05:09.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:05:09.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:05:09.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:05:09.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:05:09.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:05:09.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:07' 2026-02-20T21:05:09.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:05:07' 2026-02-20T21:05:09.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:09.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:09.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:09.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:09.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:05:07' 2026-02-20T21:05:09.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:09.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:09.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:05:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtestc5fdae76-300b-4ccc-baee-3e7deda3028dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"}44114415mirror.1trial1992026-02-20 21:05:07test1b25ea358-0490-4428-8a73-d03a18264814up+stoppedstopped44044409mirror.0trial1992026-02-20 21:05:02' 2026-02-20T21:05:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:05:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:05:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:07' 2026-02-20T21:05:09.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:05:09.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:05:09.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:05:09.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:05:09.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:05:09.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:09.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:09.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ all_admin_daemons cluster1 rbd mirror start mirror cluster2-DNE 2026-02-20T21:05:09.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:09.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:09.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:05:09.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:09.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:0 rbd mirror start mirror cluster2-DNE 2026-02-20T21:05:09.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:05:09.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:09.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:09.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:09.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:05:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:05:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:05:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:05:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:09.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:09.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:09.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:09.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:09.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:09.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:09.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:09.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:09.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:09.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:09.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:09.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:09.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:09.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start mirror cluster2-DNE 2026-02-20T21:05:09.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:09.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:1 rbd mirror start mirror cluster2-DNE 2026-02-20T21:05:09.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:05:09.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:09.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:09.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:05:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:05:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:05:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:05:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:05:09.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:05:09.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:09.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:09.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:05:09.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:05:09.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:09.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:09.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:09.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:09.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:09.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:09.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:09.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:05:09.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:09.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:05:09.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:09.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:09.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:09.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror start mirror cluster2-DNE 2026-02-20T21:05:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:2 rbd mirror start mirror cluster2-DNE 2026-02-20T21:05:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:05:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:05:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:2 2026-02-20T21:05:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=2 2026-02-20T21:05:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:05:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=2 2026-02-20T21:05:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:05:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:05:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:05:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:05:09.543 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:05:09.543 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:09.543 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:09.543 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:09.543 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:05:09.543 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:05:09.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:09.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:09.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:05:09.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:05:09.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:05:09.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:09.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:09.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:09.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror start mirror cluster2-DNE 2026-02-20T21:05:09.591 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:09.591 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:3 rbd mirror start mirror cluster2-DNE 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:3 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=3 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=3 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:05:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:05:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror start mirror cluster2-DNE 2026-02-20T21:05:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-20T21:05:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:09.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-20T21:05:09.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:09.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:09.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:09.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:05:09.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:05:09.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:09.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:10.657 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-20T21:05:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:05:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:05:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:05:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-20T21:05:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:05:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:05:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:05:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:05:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:05:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:05:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-20T21:05:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:05:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:05:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:05:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:05:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:05:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-20T21:05:10.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:05:10.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:05:10.713 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:05:10.713 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:05:10.713 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:05:10.714 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:05:10.714 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:05:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-02-20T21:05:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:11.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-20T21:05:11.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:11.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:11.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:11.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:11.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:11.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:11.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:11.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:11.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-20T21:05:11.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test1: 2026-02-20T21:05:11.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: b25ea358-0490-4428-8a73-d03a18264814 2026-02-20T21:05:11.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:05:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621471,"remote_snapshot_timestamp":1771621471,"replay_state":"idle"} 2026-02-20T21:05:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:05:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:05:09' 2026-02-20T21:05:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:05:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:05:11.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:05:11.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621471,"remote_snapshot_timestamp":1771621471,"replay_state":"idle"} 2026-02-20T21:05:11.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:05:11.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:05:11.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:05:11.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:05:11.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:05:11.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:05:11.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:05:11.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:05:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:09' 2026-02-20T21:05:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-20 21:05:09' 2026-02-20T21:05:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:05:09' 2026-02-20T21:05:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:05:11.787 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtestc5fdae76-300b-4ccc-baee-3e7deda3028dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"}44114415mirror.1trial1992026-02-20 21:05:07test1b25ea358-0490-4428-8a73-d03a18264814up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621471,"remote_snapshot_timestamp":1771621471,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:05:09' 2026-02-20T21:05:11.787 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:05:11.787 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-20T21:05:11.788 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:09' 2026-02-20T21:05:11.789 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-20T21:05:11.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:05:11.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:05:11.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:05:11.791 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:05:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemons cluster1 rbd mirror restart mirror/test 2026-02-20T21:05:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_instance=cluster1 2026-02-20T21:05:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance=cluster1 2026-02-20T21:05:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loop_instance 2026-02-20T21:05:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:05:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:05:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:0 rbd mirror restart mirror/test 2026-02-20T21:05:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:05:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:05:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:05:11.798 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:11.798 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:11.798 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:11.798 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:05:11.798 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:11.798 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:11.798 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:05:11.798 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:11.798 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:11.798 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:11.798 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:11.799 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:11.799 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:11.799 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:11.799 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:11.799 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:11.799 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:11.799 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:11.799 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:11.799 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:11.799 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:11.805 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:11.806 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:11.806 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:11.806 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart mirror/test 2026-02-20T21:05:11.847 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:no valid command found; 1 closest matches: 2026-02-20T21:05:11.847 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror restart mirror/test1 2026-02-20T21:05:11.848 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:05:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:1 rbd mirror restart mirror/test 2026-02-20T21:05:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:05:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:05:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:05:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:05:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:05:11.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:11.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:05:11.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:05:11.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:11.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:11.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:05:11.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:05:11.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:11.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:11.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:11.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:11.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:11.853 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:11.853 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:11.853 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:05:11.853 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:11.853 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:05:11.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:11.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:11.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:11.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart mirror/test 2026-02-20T21:05:11.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:11.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:05:11.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:11.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:11.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:11.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:05:11.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:11.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:11.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:11.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:05:11.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:05:11.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:11.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:12.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:05:12.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:12.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:12.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:12.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:05:12.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:05:12.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:05:12.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:05:12.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:05:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:05:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:05:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:05:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:05:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:05:12.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:12.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:05:12.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:05:12.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:12.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:12.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:12.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:12.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:12.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:12.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:12.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:12.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:12.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:12.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:12.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:12.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:12.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:12.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:05:12.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:12.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:12.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:12.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:05:12.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:12.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:12.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:12.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:12.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:12.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:12.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:12.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:05:12.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:05:12.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:12.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:12.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:12.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:12.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:05:12.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:no valid command found; 1 closest matches: 2026-02-20T21:05:12.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status mirror/test1 2026-02-20T21:05:12.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:05:12.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:12.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:05:12.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:05:12.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:12.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:12.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:05:12.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:05:12.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:12.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:12.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:12.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:12.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:12.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:12.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:12.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:05:12.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:12.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:05:12.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:05:12.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:12.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:12.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:05:12.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:05:12.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:12.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:12.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:12.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:12.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:12.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:12.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:12.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:05:12.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:05:12.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:05:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:05:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:05:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:05:13.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:05:13.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:05:13.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:05:13.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:05:13.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:05:13.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:13.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:05:13.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:13.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:13.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:13.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:13.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:13.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:13.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:14.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:05:14.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:14.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:14.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:14.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:14.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:14.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:14.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:14.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:14.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:05:14.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:05:14.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: c5fdae76-300b-4ccc-baee-3e7deda3028d 2026-02-20T21:05:14.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:05:14.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"} 2026-02-20T21:05:14.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:05:14.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:05:11' 2026-02-20T21:05:14.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:05:14.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:05:14.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:05:14.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"} 2026-02-20T21:05:14.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:05:14.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:05:14.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:05:14.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:05:14.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:05:14.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:05:14.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:05:14.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:05:14.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:11' 2026-02-20T21:05:14.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:05:11' 2026-02-20T21:05:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:05:11' 2026-02-20T21:05:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:05:14.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtestc5fdae76-300b-4ccc-baee-3e7deda3028dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"}44114415mirror.1trial1992026-02-20 21:05:11test1b25ea358-0490-4428-8a73-d03a18264814up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621471,"remote_snapshot_timestamp":1771621471,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:05:09' 2026-02-20T21:05:14.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:05:14.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:05:14.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:11' 2026-02-20T21:05:14.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:05:14.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:05:14.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:05:14.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:05:14.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:05:14.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:14.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:14.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ all_admin_daemons cluster1 rbd mirror restart mirror cluster2-DNE 2026-02-20T21:05:14.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:14.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:14.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:05:14.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:14.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:0 rbd mirror restart mirror cluster2-DNE 2026-02-20T21:05:14.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:05:14.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:14.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:14.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:05:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:05:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:05:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:05:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:14.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:14.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:14.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:14.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:14.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:14.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:14.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:14.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:14.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:14.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:14.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:14.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:14.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart mirror cluster2-DNE 2026-02-20T21:05:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:1 rbd mirror restart mirror cluster2-DNE 2026-02-20T21:05:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:05:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:05:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:05:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:05:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:05:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:05:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:05:14.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:14.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:14.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:05:14.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:05:14.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:14.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:14.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:14.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:14.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:14.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:14.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:14.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:05:14.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:14.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:05:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart mirror cluster2-DNE 2026-02-20T21:05:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:2 rbd mirror restart mirror cluster2-DNE 2026-02-20T21:05:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:05:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:05:14.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:2 2026-02-20T21:05:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=2 2026-02-20T21:05:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:05:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=2 2026-02-20T21:05:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:05:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:05:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:05:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:14.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:14.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:05:14.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:05:14.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:14.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:14.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:14.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:05:14.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:05:14.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:14.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:14.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:05:14.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:05:14.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:05:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:14.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror restart mirror cluster2-DNE 2026-02-20T21:05:14.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:3 rbd mirror restart mirror cluster2-DNE 2026-02-20T21:05:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:05:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:05:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:3 2026-02-20T21:05:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=3 2026-02-20T21:05:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:05:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=3 2026-02-20T21:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:05:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:05:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:05:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:05:14.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:05:14.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:14.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:14.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:14.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror restart mirror cluster2-DNE 2026-02-20T21:05:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:05:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:05:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:05:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:05:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:15.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:05:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:05:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:05:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:05:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:05:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:05:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:05:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:05:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:15.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:05:15.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:05:15.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:15.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:05:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:05:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:05:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:05:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:05:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:05:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:05:15.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:15.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:15.353 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:15.353 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:15.353 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:05:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:no valid command found; 1 closest matches: 2026-02-20T21:05:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status mirror/test1 2026-02-20T21:05:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:05:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:05:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:05:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:05:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:05:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:05:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:05:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:05:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:05:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:05:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:05:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:05:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:05:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:05:15.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:05:15.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:05:15.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:05:15.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:05:15.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:05:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:05:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:05:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-20T21:05:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-20T21:05:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:05:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:05:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:15.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:16.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-20T21:05:16.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:16.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:16.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:16.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:05:16.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:05:16.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:05:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-20T21:05:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:05:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:05:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:05:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:05:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:05:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:16.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:05:16.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:16.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-20T21:05:16.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:05:16.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:16.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:16.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:16.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:16.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:05:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:05:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:05:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:05:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-20T21:05:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:05:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:05:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:05:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:05:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:05:16.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:05:16.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:05:16.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:16.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ all_admin_daemons cluster1 rbd mirror stop mirror cluster2-DNE 2026-02-20T21:05:16.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:16.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:16.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:05:16.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:16.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:0 rbd mirror stop mirror cluster2-DNE 2026-02-20T21:05:16.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:05:16.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:16.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:16.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:16.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:05:16.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:16.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:16.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:16.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:05:16.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:16.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:16.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:16.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:05:16.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:16.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:16.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:05:16.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:16.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:16.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:16.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:16.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:16.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:16.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:16.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:16.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:16.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:16.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:16.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:16.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:16.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:16.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:16.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:16.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:16.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop mirror cluster2-DNE 2026-02-20T21:05:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:1 rbd mirror stop mirror cluster2-DNE 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:05:16.591 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:16.591 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:16.591 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:16.591 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror stop mirror cluster2-DNE 2026-02-20T21:05:16.636 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:16.636 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:2 rbd mirror stop mirror cluster2-DNE 2026-02-20T21:05:16.636 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:05:16.636 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:16.636 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:16.636 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:05:16.636 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:2 2026-02-20T21:05:16.636 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:16.636 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:16.636 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:16.636 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=2 2026-02-20T21:05:16.636 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:05:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=2 2026-02-20T21:05:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:05:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:05:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:05:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:05:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:05:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:05:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:05:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:05:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:05:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:05:16.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:16.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:16.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:16.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror stop mirror cluster2-DNE 2026-02-20T21:05:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:3 rbd mirror stop mirror cluster2-DNE 2026-02-20T21:05:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:05:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:05:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:3 2026-02-20T21:05:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=3 2026-02-20T21:05:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:05:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=3 2026-02-20T21:05:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:05:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:05:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:05:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:05:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:05:16.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:05:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:05:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:05:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:05:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:05:16.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:16.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:16.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:16.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror stop mirror cluster2-DNE 2026-02-20T21:05:16.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:05:16.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:16.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:16.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:16.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:05:16.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:16.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:16.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:16.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:05:16.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:05:16.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:16.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:17.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:05:17.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:17.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:17.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:17.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:05:17.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:05:17.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:05:17.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:05:17.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:05:17.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:05:17.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:17.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:17.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:05:17.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:05:17.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:17.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:05:17.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:17.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:05:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:05:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:05:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:05:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:05:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:17.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:17.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:17.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:05:17.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:05:17.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:17.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:17.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:17.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:17.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:05:17.806 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:no valid command found; 1 closest matches: 2026-02-20T21:05:17.806 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status mirror/test1 2026-02-20T21:05:17.806 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:05:17.810 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:17.810 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:05:17.810 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:05:17.810 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:17.810 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:17.810 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:05:17.810 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:05:17.810 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:17.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:17.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:17.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:17.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:17.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:17.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:17.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:05:17.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:17.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:05:17.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:05:17.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:17.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:17.812 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:05:17.812 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:05:17.812 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:17.812 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:17.812 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:17.812 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:17.812 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:17.812 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:17.812 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:17.812 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:05:17.812 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:05:17.812 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:05:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:05:17.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:05:17.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:05:17.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:05:17.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:05:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:05:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test1 2026-02-20T21:05:17.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:17.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:17.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:17.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test1 stopped 2026-02-20T21:05:17.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:17.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:17.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:17.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:05:17.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:05:17.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:17.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test1 stopped 2026-02-20T21:05:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:05:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:05:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:05:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-20T21:05:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:05:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:05:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:05:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:05:18.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:05:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:18.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:05:18.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:18.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-20T21:05:18.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:05:18.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:18.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:18.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:18.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:18.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:18.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:18.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:18.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:18.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:18.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:18.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:18.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:18.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:18.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:18.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:05:18.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:18.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:05:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:05:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:05:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:18.882 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:18.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:18.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:18.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-20T21:05:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:05:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:05:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:05:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:05:18.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:05:18.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:18.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:05:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:05:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:05:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:05:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:05:19.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:05:19.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: c5fdae76-300b-4ccc-baee-3e7deda3028d 2026-02-20T21:05:19.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:05:19.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: stopped 2026-02-20T21:05:19.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:05:19.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:05:16' 2026-02-20T21:05:19.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:05:19.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:05:19.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:05:19.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: stopped 2026-02-20T21:05:19.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:05:19.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:05:19.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:05:19.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:05:19.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:05:19.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:05:19.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:05:19.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:05:19.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:16' 2026-02-20T21:05:19.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:05:16' 2026-02-20T21:05:19.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:19.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:19.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:19.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:05:19.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:05:16' 2026-02-20T21:05:19.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:19.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:19.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:05:20.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtestc5fdae76-300b-4ccc-baee-3e7deda3028dup+stoppedstopped44114415mirror.1trial1992026-02-20 21:05:16test1b25ea358-0490-4428-8a73-d03a18264814up+stoppedstopped44044409mirror.0trial1992026-02-20 21:05:16' 2026-02-20T21:05:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:05:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:05:20.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:16' 2026-02-20T21:05:20.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:05:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:05:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:05:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:05:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:05:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+stopped 2026-02-20T21:05:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:05:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+stopped '' '' 2026-02-20T21:05:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:05:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:21.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:21.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:21.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-20T21:05:21.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test1: 2026-02-20T21:05:21.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: b25ea358-0490-4428-8a73-d03a18264814 2026-02-20T21:05:21.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:05:21.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: stopped 2026-02-20T21:05:21.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:05:21.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:05:16' 2026-02-20T21:05:21.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:05:21.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:05:21.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:05:21.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: stopped 2026-02-20T21:05:21.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:05:21.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:05:21.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:05:21.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:05:21.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:05:21.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:05:21.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:05:21.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:05:21.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:16' 2026-02-20T21:05:21.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+stopped '2026-02-20 21:05:16' 2026-02-20T21:05:21.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:21.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:21.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:21.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:05:21.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:05:16' 2026-02-20T21:05:21.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:21.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:21.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:05:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtestc5fdae76-300b-4ccc-baee-3e7deda3028dup+stoppedstopped44114415mirror.1trial1992026-02-20 21:05:16test1b25ea358-0490-4428-8a73-d03a18264814up+stoppedstopped44044409mirror.0trial1992026-02-20 21:05:16' 2026-02-20T21:05:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:05:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-20T21:05:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:16' 2026-02-20T21:05:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-20T21:05:21.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:05:21.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:05:21.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:05:21.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:05:21.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:21.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:21.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ all_admin_daemons cluster1 rbd mirror restart mirror cluster2-DNE 2026-02-20T21:05:21.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:21.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:21.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:05:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:0 rbd mirror restart mirror cluster2-DNE 2026-02-20T21:05:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:05:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:05:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:05:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:05:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:05:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:21.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:21.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:21.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:21.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:21.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:21.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart mirror cluster2-DNE 2026-02-20T21:05:21.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:21.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:1 rbd mirror restart mirror cluster2-DNE 2026-02-20T21:05:21.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:05:21.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:05:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:05:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:05:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:05:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:05:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:05:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:05:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:05:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:05:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:21.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:05:21.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:21.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:21.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:21.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart mirror cluster2-DNE 2026-02-20T21:05:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:2 rbd mirror restart mirror cluster2-DNE 2026-02-20T21:05:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:05:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:21.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:05:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:2 2026-02-20T21:05:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=2 2026-02-20T21:05:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:05:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=2 2026-02-20T21:05:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:05:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:05:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:05:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:05:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:05:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:05:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:05:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:05:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:05:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:05:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror restart mirror cluster2-DNE 2026-02-20T21:05:21.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:21.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:3 rbd mirror restart mirror cluster2-DNE 2026-02-20T21:05:21.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:05:21.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:21.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:21.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:05:21.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:3 2026-02-20T21:05:21.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:21.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:21.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:21.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=3 2026-02-20T21:05:21.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:05:21.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:21.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:21.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=3 2026-02-20T21:05:21.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:05:21.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:05:21.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:05:21.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:05:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:05:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:05:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:05:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:05:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:05:21.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:05:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror restart mirror cluster2-DNE 2026-02-20T21:05:21.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:05:21.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:21.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:21.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:21.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:05:21.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:21.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:21.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:21.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:05:21.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:05:21.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:21.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:22.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:05:22.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:22.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:22.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:22.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:05:22.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:05:22.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:05:22.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:05:22.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:05:22.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:22.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:22.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:05:22.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:05:22.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:22.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:05:22.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:05:22.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:22.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:05:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:05:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:05:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:22.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:22.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:22.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:22.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:22.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:22.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:22.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:22.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:22.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:05:22.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:22.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:22.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:22.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:05:22.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:22.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:22.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:22.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:22.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:22.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:22.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:22.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:05:22.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:05:22.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:22.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:22.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:22.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:22.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:05:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:no valid command found; 1 closest matches: 2026-02-20T21:05:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status mirror/test1 2026-02-20T21:05:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:05:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:05:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:05:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:05:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:05:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:05:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:05:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:05:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:05:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:05:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:05:22.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:05:22.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:05:22.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:05:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:05:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:05:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:05:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:05:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:05:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:05:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:05:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-20T21:05:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-20T21:05:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:05:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:05:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-20T21:05:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:05:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:05:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:05:23.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-20T21:05:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:05:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:05:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:05:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:05:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:05:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:05:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-20T21:05:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:05:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:05:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:05:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:23.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:05:23.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:05:23.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:23.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:23.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:23.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:23.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-20T21:05:23.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:05:23.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:05:23.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:05:23.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:05:23.507 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:05:23.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:05:23.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:05:23.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:23.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:05:23.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:23.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:23.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:23.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:23.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:23.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:23.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:23.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:24.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:05:24.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:24.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:24.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:24.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:24.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:24.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:24.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:24.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:24.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:05:24.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:05:24.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: c5fdae76-300b-4ccc-baee-3e7deda3028d 2026-02-20T21:05:24.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:05:24.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"} 2026-02-20T21:05:24.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:05:24.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:05:21' 2026-02-20T21:05:24.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:05:24.541 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:05:24.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:05:24.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"} 2026-02-20T21:05:24.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:05:24.543 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:05:24.543 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:05:24.543 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:05:24.543 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:05:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:05:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:05:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:05:24.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:21' 2026-02-20T21:05:24.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:05:21' 2026-02-20T21:05:24.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:24.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:24.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:24.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:24.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:05:21' 2026-02-20T21:05:24.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:24.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:24.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:05:24.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtestc5fdae76-300b-4ccc-baee-3e7deda3028dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"}44114415mirror.1trial1992026-02-20 21:05:21test1b25ea358-0490-4428-8a73-d03a18264814up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621471,"remote_snapshot_timestamp":1771621471,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:05:21' 2026-02-20T21:05:24.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:05:24.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:05:24.581 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:21' 2026-02-20T21:05:24.581 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:05:24.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:05:24.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:05:24.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:05:24.584 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:05:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-02-20T21:05:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:24.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:24.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:24.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:24.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:24.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:24.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:24.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:25.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-20T21:05:25.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:25.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:25.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:25.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:25.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:25.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:25.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:25.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:25.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-20T21:05:25.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test1: 2026-02-20T21:05:25.618 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: b25ea358-0490-4428-8a73-d03a18264814 2026-02-20T21:05:25.618 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:05:25.618 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621471,"remote_snapshot_timestamp":1771621471,"replay_state":"idle"} 2026-02-20T21:05:25.618 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:05:25.618 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:05:21' 2026-02-20T21:05:25.618 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:05:25.619 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:05:25.619 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:05:25.619 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621471,"remote_snapshot_timestamp":1771621471,"replay_state":"idle"} 2026-02-20T21:05:25.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:05:25.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:05:25.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:05:25.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:05:25.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:05:25.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:05:25.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:05:25.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:05:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:21' 2026-02-20T21:05:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-20 21:05:21' 2026-02-20T21:05:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:05:21' 2026-02-20T21:05:25.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:25.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:25.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:05:25.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtestc5fdae76-300b-4ccc-baee-3e7deda3028dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"}44114415mirror.1trial1992026-02-20 21:05:21test1b25ea358-0490-4428-8a73-d03a18264814up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621471,"remote_snapshot_timestamp":1771621471,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:05:21' 2026-02-20T21:05:25.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:05:25.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-20T21:05:25.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:21' 2026-02-20T21:05:25.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-20T21:05:25.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:05:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:05:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:05:25.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:05:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ flush cluster1 2026-02-20T21:05:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool= 2026-02-20T21:05:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image= 2026-02-20T21:05:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:05:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:05:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemons cluster1 rbd mirror flush 2026-02-20T21:05:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_instance=cluster1 2026-02-20T21:05:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance=cluster1 2026-02-20T21:05:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loop_instance 2026-02-20T21:05:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:05:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:05:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:0 rbd mirror flush 2026-02-20T21:05:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:05:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:05:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:05:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:25.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:05:25.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:25.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:25.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:05:25.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:25.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:25.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:25.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:25.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:25.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:25.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:25.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:25.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:25.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:25.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:25.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:25.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:25.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:25.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:25.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:25.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:25.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush 2026-02-20T21:05:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ all_admin_daemons cluster1 rbd mirror status 2026-02-20T21:05:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:05:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:0 rbd mirror status 2026-02-20T21:05:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:05:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:05:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:05:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:05:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:05:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:25.735 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:25.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:25.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:25.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status 2026-02-20T21:05:25.776 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:{ 2026-02-20T21:05:25.776 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "pool_replayers": [ 2026-02-20T21:05:25.776 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "peer": "uuid: 051657c0-214b-4bab-85b7-d4f8366fff9d cluster: cluster2-DNE client: client.mirror", 2026-02-20T21:05:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "pool": "mirror", 2026-02-20T21:05:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "instance_id": "4409", 2026-02-20T21:05:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "stopped (manual)", 2026-02-20T21:05:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "leader_instance_id": "4409", 2026-02-20T21:05:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "leader": true, 2026-02-20T21:05:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "instances": [ 2026-02-20T21:05:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "4409", 2026-02-20T21:05:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "4415", 2026-02-20T21:05:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "4425", 2026-02-20T21:05:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "4433" 2026-02-20T21:05:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: ], 2026-02-20T21:05:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.0.50691.cluster1.94243397466496.asok", 2026-02-20T21:05:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50691.cluster2-DNE.94243410213248.asok", 2026-02-20T21:05:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "sync_throttler": { 2026-02-20T21:05:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "max_parallel_requests": 5, 2026-02-20T21:05:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "running_requests": 0, 2026-02-20T21:05:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "waiting_requests": 0 2026-02-20T21:05:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "deletion_throttler": { 2026-02-20T21:05:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "max_parallel_requests": 1, 2026-02-20T21:05:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "running_requests": 0, 2026-02-20T21:05:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "waiting_requests": 0 2026-02-20T21:05:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "namespace_replayers": [ 2026-02-20T21:05:25.779 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.779 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_namespace": "", 2026-02-20T21:05:25.779 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_namespace": "", 2026-02-20T21:05:25.779 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_replayers": [ 2026-02-20T21:05:25.779 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.779 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "name": "mirror/test1", 2026-02-20T21:05:25.779 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:05:25.779 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.779 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: ], 2026-02-20T21:05:25.779 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_deleter": { 2026-02-20T21:05:25.779 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_deleter_status": { 2026-02-20T21:05:25.779 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "delete_images_queue": [], 2026-02-20T21:05:25.779 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "failed_deletes_queue": [] 2026-02-20T21:05:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_namespace": "ns1", 2026-02-20T21:05:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_namespace": "ns1", 2026-02-20T21:05:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_replayers": [], 2026-02-20T21:05:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_deleter": { 2026-02-20T21:05:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_deleter_status": { 2026-02-20T21:05:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "delete_images_queue": [], 2026-02-20T21:05:25.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "failed_deletes_queue": [] 2026-02-20T21:05:25.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_namespace": "ns2", 2026-02-20T21:05:25.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_namespace": "ns2", 2026-02-20T21:05:25.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_replayers": [], 2026-02-20T21:05:25.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_deleter": { 2026-02-20T21:05:25.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_deleter_status": { 2026-02-20T21:05:25.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "delete_images_queue": [], 2026-02-20T21:05:25.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "failed_deletes_queue": [] 2026-02-20T21:05:25.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: ] 2026-02-20T21:05:25.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "peer": "uuid: 09866fd0-f275-49ba-8515-82be1eebfde5 cluster: cluster2-DNE client: client.mirror", 2026-02-20T21:05:25.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "pool": "mirror_parent", 2026-02-20T21:05:25.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "instance_id": "4420", 2026-02-20T21:05:25.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "stopped (manual)", 2026-02-20T21:05:25.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "leader_instance_id": "4420", 2026-02-20T21:05:25.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "leader": true, 2026-02-20T21:05:25.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "instances": [ 2026-02-20T21:05:25.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "4420", 2026-02-20T21:05:25.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "4430", 2026-02-20T21:05:25.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "4436", 2026-02-20T21:05:25.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "4439" 2026-02-20T21:05:25.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: ], 2026-02-20T21:05:25.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.0.50691.cluster1.94243401488768.asok", 2026-02-20T21:05:25.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50691.cluster2-DNE.94243404265856.asok", 2026-02-20T21:05:25.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "sync_throttler": { 2026-02-20T21:05:25.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "max_parallel_requests": 5, 2026-02-20T21:05:25.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "running_requests": 0, 2026-02-20T21:05:25.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "waiting_requests": 0 2026-02-20T21:05:25.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "deletion_throttler": { 2026-02-20T21:05:25.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "max_parallel_requests": 1, 2026-02-20T21:05:25.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "running_requests": 0, 2026-02-20T21:05:25.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "waiting_requests": 0 2026-02-20T21:05:25.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "namespace_replayers": [ 2026-02-20T21:05:25.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_namespace": "", 2026-02-20T21:05:25.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_namespace": "", 2026-02-20T21:05:25.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_replayers": [], 2026-02-20T21:05:25.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_deleter": { 2026-02-20T21:05:25.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_deleter_status": { 2026-02-20T21:05:25.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "delete_images_queue": [], 2026-02-20T21:05:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "failed_deletes_queue": [] 2026-02-20T21:05:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_namespace": "ns1", 2026-02-20T21:05:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_namespace": "ns1", 2026-02-20T21:05:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_replayers": [], 2026-02-20T21:05:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_deleter": { 2026-02-20T21:05:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_deleter_status": { 2026-02-20T21:05:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "delete_images_queue": [], 2026-02-20T21:05:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "failed_deletes_queue": [] 2026-02-20T21:05:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.786 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.786 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.786 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: ] 2026-02-20T21:05:25.786 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.786 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: ] 2026-02-20T21:05:25.786 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:} 2026-02-20T21:05:25.789 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:25.789 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:1 rbd mirror status 2026-02-20T21:05:25.789 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:05:25.789 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:25.789 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:25.789 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:05:25.789 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:05:25.789 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:25.789 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:25.789 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:25.789 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:05:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:05:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:05:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:05:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:05:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:05:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:25.791 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:25.791 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:25.791 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:25.791 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:25.791 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:05:25.791 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:25.791 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:05:25.792 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:25.792 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:25.792 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:25.792 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status 2026-02-20T21:05:25.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:{ 2026-02-20T21:05:25.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "pool_replayers": [ 2026-02-20T21:05:25.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "peer": "uuid: 051657c0-214b-4bab-85b7-d4f8366fff9d cluster: cluster2-DNE client: client.mirror", 2026-02-20T21:05:25.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "pool": "mirror", 2026-02-20T21:05:25.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "instance_id": "4415", 2026-02-20T21:05:25.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "stopped (manual)", 2026-02-20T21:05:25.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "leader_instance_id": "4409", 2026-02-20T21:05:25.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "leader": false, 2026-02-20T21:05:25.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.1.50734.cluster1.94003959208320.asok", 2026-02-20T21:05:25.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50734.cluster2-DNE.94003970505088.asok", 2026-02-20T21:05:25.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "sync_throttler": { 2026-02-20T21:05:25.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "max_parallel_requests": 5, 2026-02-20T21:05:25.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "running_requests": 0, 2026-02-20T21:05:25.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "waiting_requests": 0 2026-02-20T21:05:25.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "deletion_throttler": { 2026-02-20T21:05:25.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "max_parallel_requests": 1, 2026-02-20T21:05:25.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "running_requests": 0, 2026-02-20T21:05:25.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "waiting_requests": 0 2026-02-20T21:05:25.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "namespace_replayers": [ 2026-02-20T21:05:25.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.834 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_namespace": "", 2026-02-20T21:05:25.834 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_namespace": "", 2026-02-20T21:05:25.834 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_replayers": [ 2026-02-20T21:05:25.834 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.834 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "name": "mirror/test", 2026-02-20T21:05:25.834 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:05:25.834 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.834 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: ] 2026-02-20T21:05:25.834 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.834 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.834 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_namespace": "ns1", 2026-02-20T21:05:25.834 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_namespace": "ns1", 2026-02-20T21:05:25.834 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_replayers": [] 2026-02-20T21:05:25.835 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.835 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.835 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_namespace": "ns2", 2026-02-20T21:05:25.835 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_namespace": "ns2", 2026-02-20T21:05:25.835 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_replayers": [] 2026-02-20T21:05:25.835 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.835 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: ] 2026-02-20T21:05:25.835 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.835 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.835 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "peer": "uuid: 09866fd0-f275-49ba-8515-82be1eebfde5 cluster: cluster2-DNE client: client.mirror", 2026-02-20T21:05:25.835 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "pool": "mirror_parent", 2026-02-20T21:05:25.835 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "instance_id": "4430", 2026-02-20T21:05:25.836 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "stopped (manual)", 2026-02-20T21:05:25.836 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "leader_instance_id": "4420", 2026-02-20T21:05:25.836 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "leader": false, 2026-02-20T21:05:25.836 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.1.50734.cluster1.94003962886528.asok", 2026-02-20T21:05:25.836 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50734.cluster2-DNE.94003965647232.asok", 2026-02-20T21:05:25.836 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "sync_throttler": { 2026-02-20T21:05:25.836 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "max_parallel_requests": 5, 2026-02-20T21:05:25.836 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "running_requests": 0, 2026-02-20T21:05:25.836 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "waiting_requests": 0 2026-02-20T21:05:25.836 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.836 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "deletion_throttler": { 2026-02-20T21:05:25.836 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "max_parallel_requests": 1, 2026-02-20T21:05:25.836 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "running_requests": 0, 2026-02-20T21:05:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "waiting_requests": 0 2026-02-20T21:05:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "namespace_replayers": [ 2026-02-20T21:05:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_namespace": "", 2026-02-20T21:05:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_namespace": "", 2026-02-20T21:05:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_replayers": [] 2026-02-20T21:05:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_namespace": "ns1", 2026-02-20T21:05:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_namespace": "ns1", 2026-02-20T21:05:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_replayers": [] 2026-02-20T21:05:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: ] 2026-02-20T21:05:25.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: ] 2026-02-20T21:05:25.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:} 2026-02-20T21:05:25.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:25.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:2 rbd mirror status 2026-02-20T21:05:25.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:05:25.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:25.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:25.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:05:25.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:2 2026-02-20T21:05:25.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:25.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:25.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:25.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=2 2026-02-20T21:05:25.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:05:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=2 2026-02-20T21:05:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:05:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:05:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:05:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:05:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:05:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:25.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:05:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:05:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:05:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:05:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:05:25.847 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:25.848 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:25.848 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:25.848 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status 2026-02-20T21:05:25.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:{ 2026-02-20T21:05:25.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "pool_replayers": [ 2026-02-20T21:05:25.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "peer": "uuid: 051657c0-214b-4bab-85b7-d4f8366fff9d cluster: cluster2-DNE client: client.mirror", 2026-02-20T21:05:25.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "pool": "mirror", 2026-02-20T21:05:25.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "instance_id": "4425", 2026-02-20T21:05:25.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "stopped (manual)", 2026-02-20T21:05:25.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "leader_instance_id": "4409", 2026-02-20T21:05:25.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "leader": false, 2026-02-20T21:05:25.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.2.50796.cluster1.94231948155264.asok", 2026-02-20T21:05:25.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50796.cluster2-DNE.94231961278848.asok", 2026-02-20T21:05:25.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "sync_throttler": { 2026-02-20T21:05:25.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "max_parallel_requests": 5, 2026-02-20T21:05:25.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "running_requests": 0, 2026-02-20T21:05:25.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "waiting_requests": 0 2026-02-20T21:05:25.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "deletion_throttler": { 2026-02-20T21:05:25.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "max_parallel_requests": 1, 2026-02-20T21:05:25.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "running_requests": 0, 2026-02-20T21:05:25.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "waiting_requests": 0 2026-02-20T21:05:25.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "namespace_replayers": [ 2026-02-20T21:05:25.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_namespace": "", 2026-02-20T21:05:25.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_namespace": "", 2026-02-20T21:05:25.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_replayers": [] 2026-02-20T21:05:25.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_namespace": "ns1", 2026-02-20T21:05:25.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_namespace": "ns1", 2026-02-20T21:05:25.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_replayers": [] 2026-02-20T21:05:25.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_namespace": "ns2", 2026-02-20T21:05:25.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_namespace": "ns2", 2026-02-20T21:05:25.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_replayers": [] 2026-02-20T21:05:25.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: ] 2026-02-20T21:05:25.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "peer": "uuid: 09866fd0-f275-49ba-8515-82be1eebfde5 cluster: cluster2-DNE client: client.mirror", 2026-02-20T21:05:25.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "pool": "mirror_parent", 2026-02-20T21:05:25.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "instance_id": "4436", 2026-02-20T21:05:25.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "stopped (manual)", 2026-02-20T21:05:25.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "leader_instance_id": "4420", 2026-02-20T21:05:25.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "leader": false, 2026-02-20T21:05:25.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.2.50796.cluster1.94231951808896.asok", 2026-02-20T21:05:25.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50796.cluster2-DNE.94231954651520.asok", 2026-02-20T21:05:25.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "sync_throttler": { 2026-02-20T21:05:25.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "max_parallel_requests": 5, 2026-02-20T21:05:25.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "running_requests": 0, 2026-02-20T21:05:25.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "waiting_requests": 0 2026-02-20T21:05:25.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "deletion_throttler": { 2026-02-20T21:05:25.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "max_parallel_requests": 1, 2026-02-20T21:05:25.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "running_requests": 0, 2026-02-20T21:05:25.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "waiting_requests": 0 2026-02-20T21:05:25.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "namespace_replayers": [ 2026-02-20T21:05:25.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_namespace": "", 2026-02-20T21:05:25.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_namespace": "", 2026-02-20T21:05:25.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_replayers": [] 2026-02-20T21:05:25.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_namespace": "ns1", 2026-02-20T21:05:25.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_namespace": "ns1", 2026-02-20T21:05:25.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_replayers": [] 2026-02-20T21:05:25.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: ] 2026-02-20T21:05:25.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: ] 2026-02-20T21:05:25.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:} 2026-02-20T21:05:25.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:25.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:3 rbd mirror status 2026-02-20T21:05:25.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:05:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:05:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:3 2026-02-20T21:05:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=3 2026-02-20T21:05:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:05:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=3 2026-02-20T21:05:25.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:05:25.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:05:25.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:05:25.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:25.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:25.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:05:25.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:05:25.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:25.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:25.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:25.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:05:25.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:05:25.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:25.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:25.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:05:25.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:05:25.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:05:25.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:25.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:25.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:25.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status 2026-02-20T21:05:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:{ 2026-02-20T21:05:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "pool_replayers": [ 2026-02-20T21:05:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "peer": "uuid: 051657c0-214b-4bab-85b7-d4f8366fff9d cluster: cluster2-DNE client: client.mirror", 2026-02-20T21:05:25.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "pool": "mirror", 2026-02-20T21:05:25.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "instance_id": "4433", 2026-02-20T21:05:25.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "stopped (manual)", 2026-02-20T21:05:25.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "leader_instance_id": "4409", 2026-02-20T21:05:25.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "leader": false, 2026-02-20T21:05:25.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.3.50864.cluster1.94709740321152.asok", 2026-02-20T21:05:25.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50864.cluster2-DNE.94709752691072.asok", 2026-02-20T21:05:25.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "sync_throttler": { 2026-02-20T21:05:25.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "max_parallel_requests": 5, 2026-02-20T21:05:25.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "running_requests": 0, 2026-02-20T21:05:25.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "waiting_requests": 0 2026-02-20T21:05:25.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "deletion_throttler": { 2026-02-20T21:05:25.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "max_parallel_requests": 1, 2026-02-20T21:05:25.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "running_requests": 0, 2026-02-20T21:05:25.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "waiting_requests": 0 2026-02-20T21:05:25.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "namespace_replayers": [ 2026-02-20T21:05:25.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_namespace": "", 2026-02-20T21:05:25.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_namespace": "", 2026-02-20T21:05:25.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_replayers": [] 2026-02-20T21:05:25.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_namespace": "ns1", 2026-02-20T21:05:25.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_namespace": "ns1", 2026-02-20T21:05:25.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_replayers": [] 2026-02-20T21:05:25.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_namespace": "ns2", 2026-02-20T21:05:25.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_namespace": "ns2", 2026-02-20T21:05:25.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_replayers": [] 2026-02-20T21:05:25.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: ] 2026-02-20T21:05:25.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "peer": "uuid: 09866fd0-f275-49ba-8515-82be1eebfde5 cluster: cluster2-DNE client: client.mirror", 2026-02-20T21:05:25.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "pool": "mirror_parent", 2026-02-20T21:05:25.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "instance_id": "4439", 2026-02-20T21:05:25.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "stopped (manual)", 2026-02-20T21:05:25.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "leader_instance_id": "4420", 2026-02-20T21:05:25.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "leader": false, 2026-02-20T21:05:25.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.3.50864.cluster1.94709744368000.asok", 2026-02-20T21:05:25.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50864.cluster2-DNE.94709747169664.asok", 2026-02-20T21:05:25.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "sync_throttler": { 2026-02-20T21:05:25.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "max_parallel_requests": 5, 2026-02-20T21:05:25.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "running_requests": 0, 2026-02-20T21:05:25.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "waiting_requests": 0 2026-02-20T21:05:25.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "deletion_throttler": { 2026-02-20T21:05:25.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "max_parallel_requests": 1, 2026-02-20T21:05:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "running_requests": 0, 2026-02-20T21:05:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "waiting_requests": 0 2026-02-20T21:05:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "namespace_replayers": [ 2026-02-20T21:05:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_namespace": "", 2026-02-20T21:05:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_namespace": "", 2026-02-20T21:05:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_replayers": [] 2026-02-20T21:05:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: }, 2026-02-20T21:05:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: { 2026-02-20T21:05:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "local_namespace": "ns1", 2026-02-20T21:05:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "remote_namespace": "ns1", 2026-02-20T21:05:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "image_replayers": [] 2026-02-20T21:05:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: ] 2026-02-20T21:05:25.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: } 2026-02-20T21:05:25.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: ] 2026-02-20T21:05:25.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:} 2026-02-20T21:05:25.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster2 mirror test1 2026-02-20T21:05:25.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:25.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:25.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:25.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:05:25.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:05:25.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster2 mirror test1 2026-02-20T21:05:25.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:25.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:25.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test1 2026-02-20T21:05:25.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 snap purge mirror/test1 2026-02-20T21:05:25.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rm mirror/test1 2026-02-20T21:05:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:05:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: test image rename' 2026-02-20T21:05:27.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:05:27.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:05:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:05:27 TEST: test image rename 2026-02-20T21:05:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:05:27 TEST: test image rename 2026-02-20T21:05:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ new_name=test_RENAMED 2026-02-20T21:05:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rename_image cluster2 mirror test test_RENAMED 2026-02-20T21:05:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local new_name=test_RENAMED 2026-02-20T21:05:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rename mirror/test mirror/test_RENAMED 2026-02-20T21:05:27.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:05:27.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test_RENAMED 2026-02-20T21:05:27.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:27.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:27.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_RENAMED 2026-02-20T21:05:27.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_RENAMED 2026-02-20T21:05:29.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 12 2026-02-20T21:05:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test_RENAMED 2026-02-20T21:05:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_RENAMED 2026-02-20T21:05:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test_RENAMED started 2026-02-20T21:05:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_RENAMED 2026-02-20T21:05:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:05:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:05:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:30.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test_RENAMED started 2026-02-20T21:05:30.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:30.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:30.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_RENAMED 2026-02-20T21:05:30.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:05:30.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:05:30.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:05:30.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_RENAMED 2026-02-20T21:05:30.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:05:30.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:30.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:05:30.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:30.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:05:30.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:05:30.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:30.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:05:30.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:30.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:05:30.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:30.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_RENAMED 2026-02-20T21:05:30.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:05:30.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:30.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:30.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:30.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:30.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:30.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:30.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:30.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:30.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:30.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:30.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:30.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:30.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:05:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:05:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:05:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:05:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:30.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:30.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_RENAMED 2026-02-20T21:05:30.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_RENAMED'] 2026-02-20T21:05:30.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:05:30.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:05:30.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:30.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_RENAMED 2026-02-20T21:05:30.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:05:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:05:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:05:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:05:30.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:30.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:05:30.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:05:30.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:30.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:30.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:05:30.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:05:30.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:30.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:30.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:30.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:30.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:30.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:30.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:30.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:05:30.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:05:30.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:05:30.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:30.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:30.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:30.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_RENAMED 2026-02-20T21:05:30.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:05:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:05:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:05:30.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:05:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:05:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:05:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:05:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_RENAMED up+replaying 2026-02-20T21:05:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_RENAMED 2026-02-20T21:05:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:30.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:31.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_RENAMED up+replaying '' '' 2026-02-20T21:05:31.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:31.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:31.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_RENAMED 2026-02-20T21:05:31.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:31.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:31.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:31.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:31.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:31.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_RENAMED 2026-02-20T21:05:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_RENAMED: 2026-02-20T21:05:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: c5fdae76-300b-4ccc-baee-3e7deda3028d 2026-02-20T21:05:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:05:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"} 2026-02-20T21:05:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:05:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:05:28' 2026-02-20T21:05:31.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:05:31.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:05:31.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:05:31.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"} 2026-02-20T21:05:31.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:05:31.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:05:31.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:05:31.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:05:31.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:05:31.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:05:31.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:05:31.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:05:31.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:28' 2026-02-20T21:05:31.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_RENAMED up+replaying '2026-02-20 21:05:28' 2026-02-20T21:05:31.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:31.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:31.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_RENAMED 2026-02-20T21:05:31.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:31.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:05:28' 2026-02-20T21:05:31.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:31.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:31.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:05:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest_RENAMEDc5fdae76-300b-4ccc-baee-3e7deda3028dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621477,"remote_snapshot_timestamp":1771621477,"replay_state":"idle"}44114415mirror.1trial1992026-02-20 21:05:28' 2026-02-20T21:05:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:05:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_RENAMED'\'']/last_update' 2026-02-20T21:05:31.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:28' 2026-02-20T21:05:31.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_RENAMED'\'']/state' 2026-02-20T21:05:31.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:05:31.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:05:31.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:05:31.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:05:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemons cluster1 rbd mirror status mirror/test_RENAMED 2026-02-20T21:05:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_instance=cluster1 2026-02-20T21:05:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance=cluster1 2026-02-20T21:05:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loop_instance 2026-02-20T21:05:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:05:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:31.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror/test_RENAMED 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:31.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:31.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:31.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:31.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:31.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:31.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:31.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:31.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_RENAMED 2026-02-20T21:05:31.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_RENAMED'] 2026-02-20T21:05:31.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:05:31.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:05:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror/test_RENAMED 2026-02-20T21:05:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:05:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:05:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:05:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:05:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:05:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:05:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:05:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:05:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:05:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:05:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:05:31.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:31.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:31.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:31.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_RENAMED 2026-02-20T21:05:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:{ 2026-02-20T21:05:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "name": "mirror/test_RENAMED", 2026-02-20T21:05:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:05:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:} 2026-02-20T21:05:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemons cluster1 rbd mirror restart mirror/test_RENAMED 2026-02-20T21:05:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_instance=cluster1 2026-02-20T21:05:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance=cluster1 2026-02-20T21:05:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loop_instance 2026-02-20T21:05:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:05:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:05:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:0 rbd mirror restart mirror/test_RENAMED 2026-02-20T21:05:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:05:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:05:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:05:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:05:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:05:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:31.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:31.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:31.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:31.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:31.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:31.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:31.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:31.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:31.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:31.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:31.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:31.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart mirror/test_RENAMED 2026-02-20T21:05:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_RENAMED'] 2026-02-20T21:05:31.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror restart : restart rbd mirror 2026-02-20T21:05:31.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:05:31.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:31.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:1 rbd mirror restart mirror/test_RENAMED 2026-02-20T21:05:31.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:05:31.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:31.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:31.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:05:31.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:05:31.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:31.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:31.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:05:31.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:05:31.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:31.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:05:31.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:31.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:05:31.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:31.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:05:31.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:05:31.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:31.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:31.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:05:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:05:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:05:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:05:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:05:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:05:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:31.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:05:31.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:31.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:31.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:31.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart mirror/test_RENAMED 2026-02-20T21:05:31.507 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:31.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test_RENAMED 2026-02-20T21:05:31.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:31.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:31.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_RENAMED 2026-02-20T21:05:31.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test_RENAMED started 2026-02-20T21:05:31.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:31.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:31.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_RENAMED 2026-02-20T21:05:31.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:05:31.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:05:31.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:31.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test_RENAMED started 2026-02-20T21:05:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_RENAMED 2026-02-20T21:05:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:05:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:05:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:05:32.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_RENAMED 2026-02-20T21:05:32.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:05:32.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:05:32.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:32.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:32.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:05:32.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:05:32.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:32.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:05:32.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:32.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:05:32.514 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:32.514 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_RENAMED 2026-02-20T21:05:32.514 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:05:32.514 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:32.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:32.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:32.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:32.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:32.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:32.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:32.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:32.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:32.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:32.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:32.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:32.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:32.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:32.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:05:32.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:32.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:32.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:32.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:05:32.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:32.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:32.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:32.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:32.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:32.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:32.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:32.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:05:32.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:05:32.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:32.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:32.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:32.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:32.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_RENAMED 2026-02-20T21:05:32.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_RENAMED'] 2026-02-20T21:05:32.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:05:32.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:05:32.569 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:32.569 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_RENAMED 2026-02-20T21:05:32.569 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:05:32.569 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:32.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:32.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:05:32.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:05:32.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:32.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:32.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:32.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:32.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:32.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:32.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:32.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:05:32.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:32.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:05:32.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:05:32.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:32.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:32.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:05:32.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:05:32.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:32.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:32.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:32.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:32.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:32.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:32.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:32.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:05:32.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:05:32.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:05:32.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:32.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:32.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:32.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_RENAMED 2026-02-20T21:05:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:05:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:05:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:05:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:05:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:05:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:05:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:05:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_RENAMED up+replaying 2026-02-20T21:05:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_RENAMED 2026-02-20T21:05:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:33.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_RENAMED up+replaying '' '' 2026-02-20T21:05:33.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:33.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:33.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_RENAMED 2026-02-20T21:05:33.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:33.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:33.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:33.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:33.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:33.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_RENAMED 2026-02-20T21:05:33.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_RENAMED: 2026-02-20T21:05:33.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: c5fdae76-300b-4ccc-baee-3e7deda3028d 2026-02-20T21:05:33.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:05:33.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621528,"remote_snapshot_timestamp":1771621528,"replay_state":"idle"} 2026-02-20T21:05:33.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:05:33.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:05:31' 2026-02-20T21:05:33.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:05:33.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:05:33.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:05:33.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621528,"remote_snapshot_timestamp":1771621528,"replay_state":"idle"} 2026-02-20T21:05:33.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:05:33.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:05:33.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:05:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:05:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:05:33.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:05:33.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:05:33.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:05:33.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:31' 2026-02-20T21:05:33.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_RENAMED up+replaying '2026-02-20 21:05:31' 2026-02-20T21:05:33.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:33.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:33.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_RENAMED 2026-02-20T21:05:33.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:33.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:05:31' 2026-02-20T21:05:33.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:33.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:33.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:05:33.699 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest_RENAMEDc5fdae76-300b-4ccc-baee-3e7deda3028dup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621528,"remote_snapshot_timestamp":1771621528,"replay_state":"idle"}44114415mirror.1trial1992026-02-20 21:05:31' 2026-02-20T21:05:33.699 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:05:33.699 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_RENAMED'\'']/last_update' 2026-02-20T21:05:33.700 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:31' 2026-02-20T21:05:33.701 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_RENAMED'\'']/state' 2026-02-20T21:05:33.702 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:05:33.702 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:05:33.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:05:33.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:05:33.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:33.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:33.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rename_image cluster2 mirror test_RENAMED test 2026-02-20T21:05:33.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:33.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:33.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_RENAMED 2026-02-20T21:05:33.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local new_name=test 2026-02-20T21:05:33.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rename mirror/test_RENAMED mirror/test 2026-02-20T21:05:33.737 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:05:33.737 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:05:33.737 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:33.737 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:33.737 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:33.737 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:05:35.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 13 2026-02-20T21:05:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:05:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:05:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:05:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:05:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:36.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:05:36.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:36.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:36.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:36.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:05:36.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:05:36.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:05:36.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:05:36.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:05:36.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:36.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:36.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:05:36.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:05:36.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:36.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:05:36.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:05:36.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:36.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:05:36.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:36.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:05:36.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:05:36.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:36.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:36.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:36.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:36.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:36.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:36.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:36.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:36.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:36.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:36.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:36.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:36.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:36.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:36.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:05:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:05:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:05:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:05:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:05:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:05:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:05:36.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:05:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:05:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:05:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:05:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:05:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:05:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:05:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:05:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:05:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:05:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:05:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:05:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:05:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:05:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:05:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:05:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:05:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:05:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:05:36.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:05:36.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:05:36.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: test trash move restore' 2026-02-20T21:05:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:05:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:05:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:05:36 TEST: test trash move restore 2026-02-20T21:05:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:05:36 TEST: test trash move restore 2026-02-20T21:05:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster2 mirror test 2026-02-20T21:05:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:05:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:05:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:05:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster2 info mirror/test 2026-02-20T21:05:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:05:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image_id=11112aa1b418 2026-02-20T21:05:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ trash_move cluster2 mirror test 2026-02-20T21:05:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:36.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 trash move mirror/test 2026-02-20T21:05:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror test deleted 2026-02-20T21:05:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=deleted 2026-02-20T21:05:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:05:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:05:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:05:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:05:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:05:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:05:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:05:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:05:38.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:05:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image_id= 2026-02-20T21:05:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:05:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:05:38.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test deleted '' 2026-02-20T21:05:38.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:38.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:38.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:38.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=deleted 2026-02-20T21:05:38.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:05:38.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:05:38.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:05:38.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:05:38.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:38.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:05:38.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:05:38.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:05:38.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:05:38.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:05:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id= 2026-02-20T21:05:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:05:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test deleted = deleted 2026-02-20T21:05:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ trash_restore cluster2 mirror test 11112aa1b418 snapshot 2026-02-20T21:05:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id=11112aa1b418 2026-02-20T21:05:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:05:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 trash restore mirror/11112aa1b418 2026-02-20T21:05:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:05:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:05:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror test snapshot 2026-02-20T21:05:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:05:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2026-02-20T21:05:39.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:05:39.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/test 2026-02-20T21:05:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test': 2026-02-20T21:05:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:05:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:05:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:05:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 11112aa1b418 2026-02-20T21:05:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.11112aa1b418 2026-02-20T21:05:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:05:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:05:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:05:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:05:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:04:17 2026 2026-02-20T21:05:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:04:17 2026 2026-02-20T21:05:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:04:17 2026 2026-02-20T21:05:39.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:05:39.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:05:39.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 77ff2c9c-a724-4c9b-ab62-4c2f49011f71 2026-02-20T21:05:39.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:05:39.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:05:39.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:39.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:39.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:39.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:05:39.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:39.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:39.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:39.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:05:39.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:05:39.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:39.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:05:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:05:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:05:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:05:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:05:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:05:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:05:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:05:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:05:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:05:40.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:40.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:05:40.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:05:40.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:05:40.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:40.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:40.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:40.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:40.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:05:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:05:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:05:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:05:40.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:40.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:05:40.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:05:40.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:40.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:40.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:05:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:05:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:05:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:05:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:05:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:05:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:05:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:05:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:05:40.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:05:40.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:40.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:40.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:40.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:05:40.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:05:40.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:05:40.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:05:40.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:40.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:05:40.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:05:40.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:40.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:40.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:05:40.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:05:40.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:40.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:40.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:40.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:05:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:05:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:05:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:05:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:05:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:05:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:05:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:05:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:05:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:05:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:05:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:05:40.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:05:40.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:40.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:40.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:40.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:05:40.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:05:40.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:05:40.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:05:40.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:40.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:05:40.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:05:40.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:40.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:40.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:05:40.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:05:40.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:40.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:40.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:40.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:05:40.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:05:40.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:40.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:40.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:05:40.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:05:40.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:05:40.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:05:40.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:40.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:40.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:05:40.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:05:40.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:40.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:40.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:40.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:05:40.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:05:40.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:40.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:40.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:05:40.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:05:40.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:05:40.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:40.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:40.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:40.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:05:40.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:05:40.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:05:40.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:05:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 1 2026-02-20T21:05:41.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:41.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:05:41.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:41.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:05:41.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:05:41.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:41.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:41.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:05:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:05:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:05:41.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:05:41.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:41.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:41.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:41.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:41.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:05:41.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:05:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:05:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:05:41.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:05:41.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:05:41.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:05:41.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:05:41.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:41.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: check if removed images'\'' OMAP are removed (with rbd-mirror on one cluster)' 2026-02-20T21:05:41.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:05:41.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:05:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:05:41 TEST: check if removed 'images'\''' OMAP are removed '(with' rbd-mirror on one 'cluster)' 2026-02-20T21:05:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:05:41 TEST: check if removed images' OMAP are removed (with rbd-mirror on one cluster) 2026-02-20T21:05:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster2 mirror test 2026-02-20T21:05:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:05:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:05:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster2 mirror test 2026-02-20T21:05:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 snap purge mirror/test 2026-02-20T21:05:41.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rm mirror/test 2026-02-20T21:05:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:05:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_in_omap cluster1 mirror 2026-02-20T21:05:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring status_global 2026-02-20T21:05:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local obj_name=rbd_mirroring 2026-02-20T21:05:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local filter=status_global 2026-02-20T21:05:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T21:05:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:05:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set +e 2026-02-20T21:05:42.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring status_global 2026-02-20T21:05:42.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:42.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:05:42.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:05:42.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local filter=status_global 2026-02-20T21:05:42.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-20T21:05:42.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -c status_global 2026-02-20T21:05:42.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:42.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:05:42.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:05:42.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-20T21:05:42.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 0 = 0 2026-02-20T21:05:42.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ error_code=0 2026-02-20T21:05:42.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set -e 2026-02-20T21:05:42.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T21:05:42.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:42.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring image_ 2026-02-20T21:05:42.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:42.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:42.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local obj_name=rbd_mirroring 2026-02-20T21:05:42.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local filter=image_ 2026-02-20T21:05:42.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T21:05:42.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:05:42.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set +e 2026-02-20T21:05:42.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring image_ 2026-02-20T21:05:42.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:05:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:05:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local filter=image_ 2026-02-20T21:05:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-20T21:05:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:05:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:05:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-20T21:05:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -c image_ 2026-02-20T21:05:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 0 = 0 2026-02-20T21:05:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ error_code=0 2026-02-20T21:05:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set -e 2026-02-20T21:05:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T21:05:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirror_leader image_map 2026-02-20T21:05:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local obj_name=rbd_mirror_leader 2026-02-20T21:05:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local filter=image_map 2026-02-20T21:05:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T21:05:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:05:42.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set +e 2026-02-20T21:05:42.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-20T21:05:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:05:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T21:05:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local filter=image_map 2026-02-20T21:05:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-20T21:05:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:05:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T21:05:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-20T21:05:42.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -c image_map 2026-02-20T21:05:42.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 1 = 0 2026-02-20T21:05:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ error_code=1 2026-02-20T21:05:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set -e 2026-02-20T21:05:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 -eq 0 ']' 2026-02-20T21:05:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T21:05:42.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:43.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set +e 2026-02-20T21:05:43.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-20T21:05:43.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:43.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:05:43.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T21:05:43.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local filter=image_map 2026-02-20T21:05:43.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-20T21:05:43.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:43.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -c image_map 2026-02-20T21:05:43.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:05:43.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T21:05:43.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-20T21:05:43.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 1 = 0 2026-02-20T21:05:43.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ error_code=1 2026-02-20T21:05:43.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set -e 2026-02-20T21:05:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 -eq 0 ']' 2026-02-20T21:05:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T21:05:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:05:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set +e 2026-02-20T21:05:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-20T21:05:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:05:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T21:05:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local filter=image_map 2026-02-20T21:05:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-20T21:05:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -c image_map 2026-02-20T21:05:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:05:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T21:05:45.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-20T21:05:45.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 0 = 0 2026-02-20T21:05:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ error_code=0 2026-02-20T21:05:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set -e 2026-02-20T21:05:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T21:05:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_in_omap cluster2 mirror 2026-02-20T21:05:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring status_global 2026-02-20T21:05:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local obj_name=rbd_mirroring 2026-02-20T21:05:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local filter=status_global 2026-02-20T21:05:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T21:05:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:05:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set +e 2026-02-20T21:05:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring status_global 2026-02-20T21:05:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:05:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:05:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:05:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local filter=status_global 2026-02-20T21:05:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-20T21:05:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:05:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:05:45.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:05:45.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-20T21:05:45.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -c status_global 2026-02-20T21:05:45.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 0 = 0 2026-02-20T21:05:45.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ error_code=0 2026-02-20T21:05:45.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set -e 2026-02-20T21:05:45.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T21:05:45.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:45.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring image_ 2026-02-20T21:05:45.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:45.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:45.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local obj_name=rbd_mirroring 2026-02-20T21:05:45.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local filter=image_ 2026-02-20T21:05:45.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T21:05:45.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:05:45.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set +e 2026-02-20T21:05:45.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring image_ 2026-02-20T21:05:45.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:05:45.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:05:45.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:05:45.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local filter=image_ 2026-02-20T21:05:45.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-20T21:05:45.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:05:45.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:05:45.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:05:45.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-20T21:05:45.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -c image_ 2026-02-20T21:05:45.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 0 = 0 2026-02-20T21:05:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ error_code=0 2026-02-20T21:05:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set -e 2026-02-20T21:05:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T21:05:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirror_leader image_map 2026-02-20T21:05:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local obj_name=rbd_mirror_leader 2026-02-20T21:05:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local filter=image_map 2026-02-20T21:05:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T21:05:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:05:45.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set +e 2026-02-20T21:05:45.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirror_leader image_map 2026-02-20T21:05:45.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:05:45.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:05:45.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T21:05:45.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local filter=image_map 2026-02-20T21:05:45.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ list_omap_keys cluster2 mirror rbd_mirror_leader 2026-02-20T21:05:45.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:05:45.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -c image_map 2026-02-20T21:05:45.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:05:45.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T21:05:45.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirror_leader 2026-02-20T21:05:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:error getting omap key set mirror/rbd_mirror_leader: (2) No such file or directory 2026-02-20T21:05:45.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 0 = 0 2026-02-20T21:05:45.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ error_code=0 2026-02-20T21:05:45.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set -e 2026-02-20T21:05:45.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T21:05:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image_and_enable_mirror cluster2 mirror test snapshot 2026-02-20T21:05:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:05:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:05:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:45.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image cluster2 mirror test 2026-02-20T21:05:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:05:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=128 2026-02-20T21:05:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:05:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,object-map,fast-diff mirror/test 2026-02-20T21:05:45.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:05:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:05:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror test snapshot 2026-02-20T21:05:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:05:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2026-02-20T21:05:46.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:05:46.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/test 2026-02-20T21:05:46.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test': 2026-02-20T21:05:46.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:05:46.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:05:46.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:05:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 119ca81aad17 2026-02-20T21:05:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.119ca81aad17 2026-02-20T21:05:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:05:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:05:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:05:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:05:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:05:45 2026 2026-02-20T21:05:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:05:45 2026 2026-02-20T21:05:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:05:45 2026 2026-02-20T21:05:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:05:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:05:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:05:46.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:05:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:05:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:05:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:05:46.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:05:46.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:46.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:47.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:05:47.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:05:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:05:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:05:47.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:05:47.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:05:47.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:05:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:05:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:05:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:05:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:05:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:05:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:05:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:05:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:05:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:05:47.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:05:47.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:47.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:47.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:47.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:47.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:05:47.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:05:47.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:05:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:05:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:47.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:05:47.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:05:47.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:47.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:47.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:05:47.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:05:47.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:47.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:47.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:47.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:05:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:05:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:05:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:05:47.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:05:47.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:47.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:47.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:47.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:05:47.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:05:47.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:47.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:47.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:05:47.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:05:47.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:05:47.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:47.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:47.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:05:47.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:05:47.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:05:47.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:05:47.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:05:47.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:47.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:05:47.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:05:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:05:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:05:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:05:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:05:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:05:47.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:05:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:05:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:05:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:05:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:05:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:05:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:05:47.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:47.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:47.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:05:47.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:05:47.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:05:47.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:47.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:47.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:05:47.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:05:47.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:05:47.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:05:47.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:05:47.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:47.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:05:47.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:05:47.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:47.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:47.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:05:47.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:05:47.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:47.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:47.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:47.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:05:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:05:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:05:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:05:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:05:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:05:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:05:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:05:47.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:47.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:47.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:47.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:05:47.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:05:47.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:47.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:47.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:05:47.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:05:47.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:05:47.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:47.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:47.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:05:47.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:05:47.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:05:47.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:05:47.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:05:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 1 2026-02-20T21:05:48.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:48.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:05:48.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:48.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:05:48.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:05:48.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:48.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:48.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:48.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:48.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:48.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:48.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:48.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:48.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:48.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:48.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:48.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:48.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:48.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:48.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:05:48.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:48.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:48.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:48.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:05:48.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:05:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:05:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:48.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:48.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:48.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:48.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:05:48.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:05:48.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:05:48.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:05:48.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:05:48.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:05:48.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:05:48.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:05:48.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:48.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test 100 2026-02-20T21:05:48.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:48.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:48.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:48.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:05:48.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:05:48.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:05:48.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:05:48.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:05:48.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:05:48.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:05:48.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 4761.97 bytes/sec: 19 MiB/s 2026-02-20T21:05:48.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:05:48.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:05:48.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:48.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:05:48.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:05:48.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:48.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:05:48.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:05:48.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:05:48.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:48.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:48.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:48.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:05:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 16 2026-02-20T21:05:49.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:05:49.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:05:49.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:49.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:05:49.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:05:49.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:49.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:05:49.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:05:49.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:05:49.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:05:49.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:05:49.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:05:49.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:05:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:05:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:05:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:05:49.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:05:49.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:05:49.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:05:49.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=16 2026-02-20T21:05:49.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:05:49.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:05:49.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:05:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:05:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:05:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:05:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:05:49.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:05:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=15 2026-02-20T21:05:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 16 = 15 2026-02-20T21:05:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:05:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.4 2026-02-20T21:05:49.744 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:05:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:05:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:05:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:05:49.776 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:05:49.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=15 2026-02-20T21:05:49.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 16 = 15 2026-02-20T21:05:49.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:05:49.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.8 2026-02-20T21:05:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:05:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:05:50.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:05:50.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:05:50.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:05:50.611 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=16 2026-02-20T21:05:50.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 16 = 16 2026-02-20T21:05:50.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:50.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:05:50.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:50.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:50.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:50.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:50.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:50.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:50.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:50.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:51.613 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:05:51.614 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:51.614 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:51.614 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:51.614 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:51.614 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:51.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:51.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:51.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:51.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:05:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:05:51.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:05:51.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:05:51.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621546,"replay_state":"idle"} 2026-02-20T21:05:51.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:05:51.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:05:48' 2026-02-20T21:05:51.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:05:51.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:05:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:05:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621546,"replay_state":"idle"} 2026-02-20T21:05:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:05:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:05:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:05:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:05:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:05:51.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:05:51.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:05:51.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:05:51.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:48' 2026-02-20T21:05:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:05:48' 2026-02-20T21:05:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:05:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:05:48' 2026-02-20T21:05:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:05:51.682 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621546,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:05:48' 2026-02-20T21:05:51.682 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:05:51.682 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:05:51.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:48' 2026-02-20T21:05:51.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:05:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:05:51.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:05:51.686 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:05:51.686 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:05:51.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:51.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:51.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: failover and failback' 2026-02-20T21:05:51.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:05:51.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:05:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:05:51 TEST: failover and failback 2026-02-20T21:05:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:05:51 TEST: failover and failback 2026-02-20T21:05:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirrors cluster2 2026-02-20T21:05:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:05:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster2:0 2026-02-20T21:05:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2:0 2026-02-20T21:05:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:05:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster2:0 2026-02-20T21:05:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster2 2026-02-20T21:05:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:05:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-20T21:05:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster2 2026-02-20T21:05:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:05:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:05:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:05:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster2 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:05:51.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster2:1 2026-02-20T21:05:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2:1 2026-02-20T21:05:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:05:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster2:1 2026-02-20T21:05:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster2 2026-02-20T21:05:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:05:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-20T21:05:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster2 2026-02-20T21:05:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:05:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:05:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:05:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:05:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster2 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:05:51.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:51.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster2:2 2026-02-20T21:05:51.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2:2 2026-02-20T21:05:51.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:51.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-20T21:05:51.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster2:2 2026-02-20T21:05:51.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:51.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:51.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster2 2026-02-20T21:05:51.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=2 2026-02-20T21:05:51.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-20T21:05:51.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster2 2026-02-20T21:05:51.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:05:51.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=2 2026-02-20T21:05:51.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:05:51.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:05:51.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster2 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:05:51.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:51.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster2:3 2026-02-20T21:05:51.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2:3 2026-02-20T21:05:51.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:05:51.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-20T21:05:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster2:3 2026-02-20T21:05:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:05:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:05:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster2 2026-02-20T21:05:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=3 2026-02-20T21:05:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-20T21:05:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster2 2026-02-20T21:05:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:05:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=3 2026-02-20T21:05:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:05:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:05:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster2 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:05:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test 2026-02-20T21:05:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-20T21:05:53.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:05:53.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:05:53.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:53.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:53.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:53.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:05:53.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:53.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:53.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:53.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:05:53.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:05:53.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:53.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:54.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:05:54.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:54.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:54.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:54.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:05:54.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:05:54.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:05:54.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:05:54.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:05:54.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:05:54.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:54.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:05:54.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:05:54.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:05:54.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:54.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:05:54.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:05:54.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:05:54.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:05:54.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:05:54.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:05:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:05:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:05:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:05:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:05:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:05:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:05:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:05:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:05:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:05:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:05:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:05:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:05:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:05:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:05:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:05:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:05:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:05:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:05:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:05:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:05:54.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:05:54.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:05:54.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:05:54.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:05:54.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:05:54.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:05:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:05:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:05:54.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:05:54.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:05:54.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:05:54.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:05:54.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:05:54.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:54.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:05:54.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:54.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:54.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:05:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:55.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:05:55.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:55.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:55.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:55.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:05:55.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:55.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:55.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:55.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:55.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:05:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:05:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:05:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:05:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:05:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:05:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:05:53 2026-02-20T21:05:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:05:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:05:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:05:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:05:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:05:53' 2026-02-20T21:05:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:05:55.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:05:55.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:05:55.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image demoted 2026-02-20T21:05:55.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:05:55.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:05:55.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:05:55.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:05:55.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:05:55.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:05:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:05:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:05:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:53' 2026-02-20T21:05:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:05:53' 2026-02-20T21:05:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:05:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:05:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:05:53' 2026-02-20T21:05:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:05:55.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image demoted44044409mirror.0trial1992026-02-20 21:05:53cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:05:53' 2026-02-20T21:05:55.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:05:55.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:05:55.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:53' 2026-02-20T21:05:55.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:05:55.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:05:55.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:05:55.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:05:55.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:05:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:05:55.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:05:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:05:56.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:05:56.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:56.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:56.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:56.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:05:56.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:56.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:56.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:56.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:56.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:05:56.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:05:56.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:05:56.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:05:56.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:05:56.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:05:53 2026-02-20T21:05:56.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:05:56.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:05:56.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:05:56.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:05:56.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:05:53' 2026-02-20T21:05:56.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:05:56.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:05:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:05:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:05:56.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:05:56.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:05:56.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:05:56.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:05:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:05:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 1 2026-02-20T21:05:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:05:56.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:05:58.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:05:58.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:58.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:58.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:58.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:05:58.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:05:58.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:05:58.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:58.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:58.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:05:58.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:05:58.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:05:58.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:05:58.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:05:58.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:05:58.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:05:56 2026-02-20T21:05:58.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:05:58.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:05:58.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:05:58.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:05:58.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:05:58' 2026-02-20T21:05:58.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:05:58.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:05:58.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:05:58.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:05:58.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:05:58.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:05:58.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:05:58.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:05:58.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:05:58.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:05:58.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:05:58.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:05:58.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:56' 2026-02-20T21:05:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:05:56' 2026-02-20T21:05:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:05:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:05:56' 2026-02-20T21:05:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:05:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:05:58.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:05:58.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:05:56cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:05:58' 2026-02-20T21:05:58.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:05:58.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:05:58.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:05:56' 2026-02-20T21:05:58.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:05:58.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:05:58.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:05:58.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:05:58.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:05:58.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:58.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:05:58.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster2 mirror test 2026-02-20T21:05:58.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:05:58.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:05:58.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:05:58.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:05:58.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-20T21:06:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:06:00.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:06:00.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:00.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:00.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:00.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:06:00.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:00.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:00.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:00.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:06:00.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:06:00.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:00.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:06:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:06:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:06:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:06:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:06:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:06:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:06:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:06:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:06:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:06:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:06:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:06:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:06:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:06:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:06:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:06:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:06:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:06:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:06:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:06:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:06:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:06:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:06:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:06:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:06:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:06:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:06:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:06:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:06:01.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:06:01.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:06:01.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:01.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:01.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:06:01.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:01.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:01.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:01.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:01.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:06:01.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:06:01.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:06:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:06:01.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:06:01.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:06:01.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:06:01.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:06:01.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:06:01.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:01.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:06:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:06:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:06:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:06:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:06:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:06:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:06:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:06:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:06:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:06:03.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:06:03.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:03.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:06:03.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:06:03.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:06:03.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:03.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:06:03.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:06:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:06:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:06:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:06:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:06:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:06:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:06:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:06:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:06:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:06:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:06:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:06:03.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:03.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:03.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:03.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:06:03.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:06:03.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:06:03.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:06:03.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:06:03.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:06:03.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:03.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:06:07.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:06:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:06:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:06:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:06:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:06:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:06:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:07.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:06:07.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:06:07.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:06:07.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:07.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:06:07.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:06:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:06:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:06:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:06:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:06:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:06:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:06:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:06:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:06:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:06:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:06:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:06:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:06:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:06:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:06:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:06:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:06:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:06:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:06:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:06:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:06:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:06:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:06:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:06:07.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:06:07.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:06:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:06:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:06:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:06:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:06:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:06:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:06:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test 100 2026-02-20T21:06:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:06:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:06:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:06:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:06:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:06:07.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:06:07.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:06:07.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 4761.97 bytes/sec: 19 MiB/s 2026-02-20T21:06:07.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:06:07.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:06:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:06:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:06:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:06:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:06:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:06:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:06:09.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 19 2026-02-20T21:06:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:06:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:06:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:06:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:06:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:06:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:06:09.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:06:09.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:06:09.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:06:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:06:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:06:09.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:06:09.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:06:09.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:06:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:06:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:06:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:06:09.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=19 2026-02-20T21:06:09.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:06:09.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:06:09.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:06:09.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:06:09.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:09.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:09.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:09.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:06:09.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:06:09.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:06:09.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:06:09.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=19 2026-02-20T21:06:09.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 19 = 19 2026-02-20T21:06:09.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:09.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-20T21:06:09.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:09.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:09.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:09.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:06:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:06:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-20T21:06:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:06:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:06:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:06:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:06:10.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:06:10.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:06:10.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:06:10.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:06:10.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:06:10.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:06 2026-02-20T21:06:10.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:06:10.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:06:10.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:06:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621552,"remote_snapshot_timestamp":1771621559,"replay_state":"idle"} 2026-02-20T21:06:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:03 2026-02-20T21:06:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:06:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 18 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.f6798963-0d74-425b-b781-90381c6de79b (peer_uuids:[]) 2026-02-20T21:06:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 19 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.55c966e4-b8c0-4a64-8258-905ef47c07ce (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:06:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:06:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:06:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:06:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:06:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:06:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:06:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:06:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:06:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:06:10.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:06:10.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:06:10.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:06:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:06' 2026-02-20T21:06:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-20 21:06:06' 2026-02-20T21:06:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:06:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:06:06' 2026-02-20T21:06:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:06:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:06:10.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:06:10.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:06:06cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621552,"remote_snapshot_timestamp":1771621559,"replay_state":"idle"}2026-02-20 21:06:03' 2026-02-20T21:06:10.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:06:10.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:06:10.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:06' 2026-02-20T21:06:10.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:06:10.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:06:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:06:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:06:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:06:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-20T21:06:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:06:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:06:10.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:10.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:10.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:10.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:06:10.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:10.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:10.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:10.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:06:11.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:06:11.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:11.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:11.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:11.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:06:11.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:11.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:11.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:06:11.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:06:11.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:06:11.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:06:11.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:06:11.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:06:11.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621552,"remote_snapshot_timestamp":1771621559,"replay_state":"idle"} 2026-02-20T21:06:11.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:06:11.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:03 2026-02-20T21:06:11.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:06:11.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:06:11.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:06:11.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:06:11.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:06' 2026-02-20T21:06:11.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:06:11.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:06:11.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:06:11.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621552,"remote_snapshot_timestamp":1771621559,"replay_state":"idle"} 2026-02-20T21:06:11.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:06:11.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:06:11.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:06:11.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:06:11.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:06:11.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:06:11.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:06:11.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:06:11.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:03' 2026-02-20T21:06:11.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:06:03' 2026-02-20T21:06:11.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:11.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:11.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:11.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:06:11.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:06:03' 2026-02-20T21:06:11.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:06:11.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:06:11.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:06:11.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621552,"remote_snapshot_timestamp":1771621559,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:06:03cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:06:06' 2026-02-20T21:06:11.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:06:11.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:06:11.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:03' 2026-02-20T21:06:11.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:06:11.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:06:11.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:06:11.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:06:11.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:06:11.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:11.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:11.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror mirror test 2026-02-20T21:06:11.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:06:11.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:06:11.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:11.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:06:11.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:06:11.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:11.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.export 2026-02-20T21:06:11.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.export 2026-02-20T21:06:11.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:06:11.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export 2026-02-20T21:06:11.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.export 2026-02-20T21:06:11.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.export 2026-02-20T21:06:11.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:06:11.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:06:11.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:06:11.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export 2026-02-20T21:06:11.581 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:06:11.585 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:06:11.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:06:11.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:06:11.721 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:06:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test 2026-02-20T21:06:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-20T21:06:13.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:06:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:06:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:06:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:13.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:13.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:13.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:06:13.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:06:13.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:13.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:06:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:06:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:06:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:06:14.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:06:14.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:06:14.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:06:14.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:06:14.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:14.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:06:14.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:06:14.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:06:14.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:14.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:06:14.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:06:14.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:06:14.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:14.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:06:14.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:06:14.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:14.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:06:14.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:06:14.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:06:14.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:14.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:06:14.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:06:14.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:06:14.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:14.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:06:14.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:06:14.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:06:14.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:14.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:06:14.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:06:14.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:06:14.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:06:14.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:06:14.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:06:14.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:14.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:14.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:06:14.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:14.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:14.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:14.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:14.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:06:14.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:06:14.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:06:14.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:14.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:14.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:14.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:06:14.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:06:14.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:06:14.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:06:14.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:06:14.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:06:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:06:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:06:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:06:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:06:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:06:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:06:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:06:15.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:06:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:06:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:06:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:06:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:06:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:06:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:13 2026-02-20T21:06:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:06:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:06:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:06:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:06:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:11' 2026-02-20T21:06:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:06:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:06:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:06:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image demoted 2026-02-20T21:06:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:06:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:06:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:06:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:06:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:06:15.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:06:15.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:06:15.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:06:15.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:13' 2026-02-20T21:06:15.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:06:13' 2026-02-20T21:06:15.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:15.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:15.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:15.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:06:15.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:06:13' 2026-02-20T21:06:15.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:06:15.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:06:15.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:06:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image demoted44044409mirror.0trial1992026-02-20 21:06:13cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:06:11' 2026-02-20T21:06:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:06:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:06:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:13' 2026-02-20T21:06:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:06:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:06:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:06:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:06:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:06:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:15.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:15.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:06:15.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:15.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:15.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:15.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:06:15.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:15.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:15.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:15.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:06:16.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:06:16.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:06:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:06:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:06:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:06:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:06:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:06:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:06:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:06:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:06:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:11 2026-02-20T21:06:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:06:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:06:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:06:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:06:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:13' 2026-02-20T21:06:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:06:16.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 1 2026-02-20T21:06:16.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:16.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:06:18.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:06:18.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:18.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:18.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:18.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:06:18.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:18.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:18.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:06:18.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:06:18.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:06:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:06:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:06:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:06:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:06:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:06:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:16 2026-02-20T21:06:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:06:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:06:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:06:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:06:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:18' 2026-02-20T21:06:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:06:18.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:06:18.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:06:18.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:06:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:06:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:06:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:06:18.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:06:18.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:06:18.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:06:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:06:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:06:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:16' 2026-02-20T21:06:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:06:16' 2026-02-20T21:06:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:06:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:06:16' 2026-02-20T21:06:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:06:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:06:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:06:18.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:06:16cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:06:18' 2026-02-20T21:06:18.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:06:18.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:06:18.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:16' 2026-02-20T21:06:18.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:06:18.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:06:18.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:06:18.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:06:18.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:06:18.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:18.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:18.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test 2026-02-20T21:06:18.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:18.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:18.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:18.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:06:18.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-20T21:06:18.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:06:18.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-20T21:06:18.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:18.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:18.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:18.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-20T21:06:18.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:18.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:18.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:18.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:06:18.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:06:18.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:18.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:06:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:06:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:06:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:06:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:06:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:06:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:06:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:06:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:06:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:06:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:06:19.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:19.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:06:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:06:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:06:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:06:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:06:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:06:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:06:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:06:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:06:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:06:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:06:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:06:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:06:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:06:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:06:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:06:19.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:06:19.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:06:19.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:06:19.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:06:19.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:19.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:19.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:19.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:19.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:06:19.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:06:19.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:06:19.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:06:19.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:06:19.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:06:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:06:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:06:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:06:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:06:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:06:20.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:20.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:06:20.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:06:20.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:20.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:06:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:06:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:06:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:06:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:06:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:06:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:06:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:06:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:06:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:06:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:06:20.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:06:20.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:06:20.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:06:20.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:06:20.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:06:20.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:20.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:20.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:20.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:06:20.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:06:20.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:06:20.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:06:20.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:06:20.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:06:20.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:06:20.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:06:20.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:06:20.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:06:20.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:06:20.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:06:20.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:06:20.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:06:20.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:06:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:06:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:20.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:06:22.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:06:22.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:22.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:22.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:22.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:06:22.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:06:22.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:06:22.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:06:22.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:06:22.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:06:22.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:22.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:06:22.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:06:22.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:06:22.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:22.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:06:22.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:06:22.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:06:22.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:22.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:06:22.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:06:22.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:22.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:06:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:06:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:06:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:06:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:06:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:06:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:06:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:06:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:06:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:06:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:06:22.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:06:22.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:06:22.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:06:22.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:06:22.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:22.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:22.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:22.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:22.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:06:22.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:06:22.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:06:22.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:06:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:06:22.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:06:22.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:06:22.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:06:22.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:06:22.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:06:22.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:22.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:22.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:06:22.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:22.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:06:22.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:06:22.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:22.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:06:22.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:06:22.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:06:22.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:06:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:06:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:06:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:06:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:06:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:06:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:06:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:06:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:06:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:06:22.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:06:22.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:06:22.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:06:22.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:22.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:22.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:22.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:06:22.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:06:22.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:06:22.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:06:22.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:06:22.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:06:22.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:06:22.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:06:22.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:06:22.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:06:22.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:06:22.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:06:22.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:06:22.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:06:22.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:06:22.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:06:22.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:06:22.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:06:22.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:22.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster1 mirror test 2026-02-20T21:06:22.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:22.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:22.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:22.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-20T21:06:25.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:06:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-20T21:06:25.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:25.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:25.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:25.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-20T21:06:25.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:25.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:25.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:25.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:06:25.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:06:25.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:25.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:06:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:06:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:06:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:06:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:06:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:06:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:06:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:06:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:06:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:06:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:06:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:06:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:06:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:06:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:06:26.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:06:27.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:06:27.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:06:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:06:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:06:27.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:27.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:27.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:06:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:06:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:06:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:06:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:06:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:06:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:06:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:06:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:06:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:06:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:06:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:06:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:06:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:06:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:06:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:06:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:06:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:06:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:06:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:27.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:27.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:06:27.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:06:27.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:06:27.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:06:27.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:06:27.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:06:27.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:06:27.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:06:27.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:06:27.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:06:27.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:06:27.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:06:27.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:06:27.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:06:27.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:06:27.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:06:27.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:27.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:06:27.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:27.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:27.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:27.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:06:27.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:27.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:27.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:27.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:06:28.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:06:28.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:28.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:28.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:28.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:06:28.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:28.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:28.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:06:28.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:06:28.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:06:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:06:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:06:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:06:28.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:06:28.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:06:28.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:23 2026-02-20T21:06:28.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:06:28.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:06:28.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:06:28.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:06:28.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:26' 2026-02-20T21:06:28.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:06:28.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:06:28.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:06:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:06:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:06:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:06:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:06:28.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:06:28.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:06:28.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:06:28.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:06:28.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:06:28.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:23' 2026-02-20T21:06:28.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:06:23' 2026-02-20T21:06:28.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:28.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:28.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:28.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:06:28.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:06:23' 2026-02-20T21:06:28.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:06:28.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:06:28.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:06:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:06:23cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:06:26' 2026-02-20T21:06:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:06:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:06:28.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:23' 2026-02-20T21:06:28.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:06:28.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:06:28.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:06:28.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:06:28.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:06:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:06:28.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:28.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:28.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:28.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:06:28.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:28.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:28.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:28.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:06:29.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:06:29.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:29.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:29.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:29.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:06:29.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:29.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:29.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:06:29.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:06:29.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:06:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:06:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:06:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:06:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:06:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:06:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:26 2026-02-20T21:06:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:06:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:06:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:06:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:06:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:28' 2026-02-20T21:06:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:06:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:06:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:06:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:06:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:06:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:06:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:06:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:06:29.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:06:29.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:06:29.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:06:29.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:06:29.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:26' 2026-02-20T21:06:29.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:06:26' 2026-02-20T21:06:29.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:29.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:29.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:29.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:06:29.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:06:26' 2026-02-20T21:06:29.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:06:29.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:06:29.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:06:29.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:06:26cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:06:28' 2026-02-20T21:06:29.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:06:29.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:06:29.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:26' 2026-02-20T21:06:29.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:06:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:06:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:06:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:06:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:06:29.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:29.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:29.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster2 mirror test 2026-02-20T21:06:29.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:29.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:29.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:29.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:06:29.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-20T21:06:31.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:06:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:06:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:06:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:31.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:31.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:06:31.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:06:31.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:31.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:06:32.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:06:32.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:32.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:32.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:32.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:06:32.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:06:32.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:06:32.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:06:32.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:06:32.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:32.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:06:32.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:06:32.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:06:32.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:32.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:06:32.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:06:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:06:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:06:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:32.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:06:32.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:06:32.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:32.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:06:32.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:06:32.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:06:32.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:32.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:06:32.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:06:32.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:06:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:06:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:06:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:06:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:06:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:06:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:06:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:06:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:06:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:06:32.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:32.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:32.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:06:32.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:32.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:32.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:32.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:32.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:06:32.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:06:32.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:06:32.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:32.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:32.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:32.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:06:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:06:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:06:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:06:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:06:32.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:06:32.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:32.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:06:34.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:06:34.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:34.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:34.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:34.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:06:34.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:06:34.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:06:34.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:06:34.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:06:34.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:06:34.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:34.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:06:34.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:06:34.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:06:34.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:34.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:06:34.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:06:34.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:06:34.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:34.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:06:34.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:06:34.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:34.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:06:34.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:06:34.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:06:34.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:34.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:06:34.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:06:34.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:06:34.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:34.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:06:34.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:06:34.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:06:34.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:34.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:06:34.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:06:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:06:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:06:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:06:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:06:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:06:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:06:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:06:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:06:34.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:34.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:34.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:34.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:06:34.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:06:34.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:06:34.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:06:34.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:06:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:06:34.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:06:34.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:06:34.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:34.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:06:34.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:06:34.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:34.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:06:34.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:06:34.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:34.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:06:34.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:06:34.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:06:34.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:34.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:34.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:34.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:06:38.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 24 2026-02-20T21:06:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:06:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:06:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:06:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:06:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:06:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:06:38.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:06:38.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:06:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:06:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:06:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:06:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:06:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:06:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:06:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:06:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:06:38.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:06:38.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=24 2026-02-20T21:06:38.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:06:38.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:06:38.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:06:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:06:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:06:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:06:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:06:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:06:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=24 2026-02-20T21:06:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 24 = 24 2026-02-20T21:06:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-20T21:06:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:06:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:06:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:06:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:06:39.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:06:39.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:39.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:39.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:39.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:06:39.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:39.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:39.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:06:39.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:06:39.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:06:39.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:06:39.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:06:39.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:06:39.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621590,"replay_state":"idle"} 2026-02-20T21:06:39.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:06:39.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:33 2026-02-20T21:06:39.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:06:39.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:06:39.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:06:39.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:06:39.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:36' 2026-02-20T21:06:39.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:06:39.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:06:39.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:06:39.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621590,"replay_state":"idle"} 2026-02-20T21:06:39.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:06:39.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:06:39.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:06:39.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:06:39.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:06:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:06:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:06:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:06:39.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:33' 2026-02-20T21:06:39.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:06:33' 2026-02-20T21:06:39.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:39.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:06:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:06:33' 2026-02-20T21:06:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:06:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:06:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:06:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621590,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:06:33cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:06:36' 2026-02-20T21:06:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:06:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:06:39.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:33' 2026-02-20T21:06:39.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:06:39.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:06:39.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:06:39.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:06:39.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:06:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-20T21:06:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:06:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:39.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:39.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:06:40.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-20T21:06:40.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:40.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:40.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:40.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:06:40.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:40.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:40.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:06:40.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:06:40.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:06:40.440 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:06:40.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:06:40.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:06:40.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:06:40.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:06:40.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:36 2026-02-20T21:06:40.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:06:40.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:06:40.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:06:40.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621590,"replay_state":"idle"} 2026-02-20T21:06:40.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:33 2026-02-20T21:06:40.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:06:40.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 24 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.a766c077-7b48-48b3-aff1-5c6c325f6964 (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:06:40.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:06:40.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:06:40.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:06:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:06:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:06:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:06:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:06:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:06:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:06:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:06:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:06:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:06:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:36' 2026-02-20T21:06:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-20 21:06:36' 2026-02-20T21:06:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:06:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:06:36' 2026-02-20T21:06:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:06:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:06:40.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:06:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:06:36cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621590,"replay_state":"idle"}2026-02-20 21:06:33' 2026-02-20T21:06:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:06:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:06:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:36' 2026-02-20T21:06:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:06:40.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:06:40.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:06:40.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:06:40.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:06:40.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:40.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror mirror test 2026-02-20T21:06:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:06:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:06:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:06:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:06:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.export 2026-02-20T21:06:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.export 2026-02-20T21:06:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:06:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export 2026-02-20T21:06:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.export 2026-02-20T21:06:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.export 2026-02-20T21:06:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:06:40.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:06:40.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:06:40.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export 2026-02-20T21:06:40.569 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:06:40.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:06:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:06:40.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:06:40.713 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:06:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test 2026-02-20T21:06:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-20T21:06:41.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:06:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:06:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:06:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:06:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:06:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:41.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:06:42.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:06:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:06:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:06:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:06:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:06:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:06:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:06:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:06:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:06:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:06:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:42.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:06:42.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:06:42.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:06:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:06:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:06:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:06:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:06:42.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:42.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:42.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:42.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:06:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:06:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:06:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:06:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:06:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:06:42.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:06:42.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:06:42.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:42.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:06:44.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:06:44.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:44.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:44.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:44.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:06:44.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:06:44.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:06:44.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:06:44.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:06:44.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:06:44.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:44.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:06:44.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:06:44.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:06:44.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:44.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:06:44.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:06:44.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:06:44.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:06:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:06:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:06:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:06:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:06:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:06:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:06:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:06:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:06:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:06:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:06:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:06:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:06:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:06:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:06:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:06:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:06:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:06:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:06:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:06:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:06:44.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:44.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:44.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:44.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:06:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:06:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:06:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:06:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:06:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:06:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:06:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:06:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:06:45.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:06:45.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:45.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:06:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:06:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:06:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:06:45.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:06:45.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:06:45.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:06:45.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:06:45.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:06:45.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:43 2026-02-20T21:06:45.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:06:45.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:06:45.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:06:45.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:06:45.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:41' 2026-02-20T21:06:45.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:06:45.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:06:45.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:06:45.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:06:45.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:06:45.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:06:45.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:06:45.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:06:45.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:06:45.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:06:45.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:06:45.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:06:45.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:43' 2026-02-20T21:06:45.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:06:43' 2026-02-20T21:06:45.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:45.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:45.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:45.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:06:45.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:06:43' 2026-02-20T21:06:45.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:06:45.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:06:45.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:06:45.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:06:43cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:06:41' 2026-02-20T21:06:45.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:06:45.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:06:45.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:43' 2026-02-20T21:06:45.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:06:45.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:06:45.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:06:45.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:06:45.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:06:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:06:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:06:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:06:46.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:06:46.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:46.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:46.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:46.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:06:46.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:46.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:46.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:06:46.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:06:46.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:06:46.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:06:46.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:06:46.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:06:46.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:06:46.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:06:46.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:41 2026-02-20T21:06:46.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:06:46.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:06:46.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:06:46.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:06:46.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:43' 2026-02-20T21:06:46.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:06:46.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:06:46.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:06:46.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:06:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:06:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:06:46.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:06:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:06:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:06:46.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:06:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:06:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:06:46.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:41' 2026-02-20T21:06:46.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:06:41' 2026-02-20T21:06:46.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:46.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:46.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:46.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:06:46.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:06:41' 2026-02-20T21:06:46.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:06:46.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:06:46.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:06:46.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:06:41cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:06:43' 2026-02-20T21:06:46.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:06:46.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:06:46.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:41' 2026-02-20T21:06:46.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:06:46.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:06:46.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:06:46.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:06:46.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:06:46.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:46.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:46.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test 2026-02-20T21:06:46.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:46.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:46.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:46.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:06:46.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-20T21:06:46.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:06:46.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-20T21:06:46.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:46.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:46.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:46.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-20T21:06:46.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:46.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:46.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:46.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:06:46.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:06:46.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:46.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:06:47.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:06:47.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:47.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:47.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:47.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:06:47.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:06:47.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:06:47.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:06:47.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:06:47.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:06:47.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:47.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:06:47.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:06:47.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:06:47.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:47.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:06:47.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:06:47.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:06:47.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:47.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:06:47.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:06:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:06:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:06:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:06:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:06:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:06:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:06:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:06:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:06:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:06:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:06:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:06:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:06:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:06:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:06:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:06:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:06:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:06:47.882 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:06:47.882 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:06:47.882 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:06:47.882 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:06:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:06:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:06:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:06:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:06:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:06:47.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:47.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:06:47.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:06:47.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:06:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:06:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:06:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:06:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:06:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:06:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:06:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:06:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:06:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:06:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:06:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:06:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:06:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:06:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:06:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:06:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:06:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:06:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:06:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:06:47.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:06:47.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:06:47.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:06:47.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:06:47.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:06:47.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:06:47.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:06:47.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:06:47.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:06:47.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:06:47.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:06:47.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:06:47.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:47.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:06:49.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:06:49.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:49.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:06:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:06:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:06:49.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:06:49.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:06:49.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:49.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:06:49.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:06:49.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:06:49.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:49.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:06:49.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:06:49.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:06:49.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:06:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:06:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:06:49.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:06:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:06:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:06:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:06:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:06:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:06:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:06:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:06:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:06:49.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:06:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:06:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:06:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:06:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:06:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:06:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:06:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:06:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:06:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:06:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:06:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:06:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:06:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:06:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:06:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:06:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:50.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:06:50.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:50.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:06:50.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:06:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:06:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:06:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:06:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:06:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:06:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:06:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:06:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:06:50.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:06:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:06:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:06:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:06:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:06:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:06:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:06:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:06:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:06:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:06:50.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:06:50.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:06:50.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:06:50.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:06:50.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:06:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:06:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:06:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:06:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:06:50.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:06:50.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:06:50.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:06:50.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:06:50.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:06:50.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:50.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:06:54.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:06:54.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:54.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:54.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:54.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:06:54.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:06:54.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:06:54.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:06:54.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:06:54.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:06:54.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:54.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:06:54.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:06:54.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:06:54.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:54.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:06:54.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:06:54.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:06:54.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:54.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:06:54.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:06:54.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:54.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:06:54.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:06:54.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:06:54.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:54.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:06:54.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:06:54.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:06:54.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:06:54.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:06:54.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:54.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:06:54.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:54.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:06:54.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:06:54.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:06:54.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:06:54.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:06:54.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:06:54.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:54.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:54.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:54.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:06:54.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:06:54.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:06:54.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:06:54.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:06:54.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:06:54.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:06:54.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:06:54.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:06:54.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:06:54.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:06:54.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:54.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:54.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:06:54.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:54.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:06:54.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:06:54.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:06:54.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:06:54.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:06:54.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:06:54.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:54.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:06:54.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:06:54.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:06:54.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:06:54.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:06:54.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:54.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:06:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:06:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:06:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:06:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:06:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:06:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:06:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:06:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:06:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:06:54.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:06:54.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:06:54.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:06:54.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:06:54.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:06:54.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:06:54.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:06:54.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:06:54.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:06:54.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:06:54.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:06:54.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:06:54.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:06:54.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:06:54.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:06:54.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:06:54.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:06:54.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:54.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster1 mirror test 100 2026-02-20T21:06:54.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:54.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:54.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:54.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:06:54.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:06:54.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:06:54.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:06:54.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:06:54.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:06:54.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:06:54.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 4545.52 bytes/sec: 18 MiB/s 2026-02-20T21:06:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:06:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:06:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:06:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:06:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:06:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:06:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-20T21:06:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-20T21:06:57.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 27 2026-02-20T21:06:57.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:06:57.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:06:57.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:57.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:06:57.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:06:57.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:57.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:06:57.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:06:57.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:06:57.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:06:57.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:06:57.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:06:57.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:06:57.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:06:57.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:06:57.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:57.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:57.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:57.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:06:57.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:06:57.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:06:57.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:06:57.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=27 2026-02-20T21:06:57.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:06:57.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:06:57.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:06:58.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:06:58.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:58.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:58.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:58.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:06:58.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:06:58.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:06:58.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:06:58.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=27 2026-02-20T21:06:58.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 27 = 27 2026-02-20T21:06:58.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:58.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:06:58.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:58.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:58.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:58.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:06:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:58.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:06:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:06:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:06:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:59.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:59.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:06:59.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:06:59.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:06:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:06:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:06:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:06:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:06:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:06:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:58 2026-02-20T21:06:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:06:59.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:06:59.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:06:59.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621606,"replay_state":"idle"} 2026-02-20T21:06:59.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:51 2026-02-20T21:06:59.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:06:59.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 27 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.9924cf36-db23-4867-ab5e-5df69756e5f4 (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:06:59.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:06:59.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:06:59.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:06:59.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:06:59.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:06:59.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:06:59.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:06:59.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:06:59.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:06:59.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:06:59.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:06:59.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:06:59.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:58' 2026-02-20T21:06:59.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:06:58' 2026-02-20T21:06:59.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:06:59.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:59.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:59.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:06:59.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:06:58' 2026-02-20T21:06:59.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:06:59.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:06:59.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:06:59.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary44044409mirror.0trial1992026-02-20 21:06:58cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621606,"replay_state":"idle"}2026-02-20 21:06:51' 2026-02-20T21:06:59.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:06:59.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:06:59.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:58' 2026-02-20T21:06:59.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:06:59.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:06:59.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:06:59.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:06:59.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:06:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:06:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster2 mirror test 2026-02-20T21:06:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:06:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-20T21:06:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:06:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:06:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:06:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:06:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:06:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:06:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:07:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-20T21:07:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:00.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:00.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:07:00.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:00.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:00.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:00.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:00.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:07:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:07:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:07:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:07:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621606,"replay_state":"idle"} 2026-02-20T21:07:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:07:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:51 2026-02-20T21:07:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:07:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:07:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:07:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:07:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:06:58' 2026-02-20T21:07:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:07:00.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:07:00.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:07:00.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621606,"replay_state":"idle"} 2026-02-20T21:07:00.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:07:00.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:07:00.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:07:00.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:07:00.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:07:00.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:07:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:07:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:07:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:51' 2026-02-20T21:07:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-20 21:06:51' 2026-02-20T21:07:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:07:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:06:51' 2026-02-20T21:07:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:00.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:07:00.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621606,"replay_state":"idle"}45294534mirror.1trial1992026-02-20 21:06:51cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:06:58' 2026-02-20T21:07:00.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:07:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:07:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:06:51' 2026-02-20T21:07:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:07:00.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:07:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:07:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:07:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:07:00.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:00.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:00.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror mirror test 2026-02-20T21:07:00.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:07:00.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:07:00.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:00.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:07:00.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:07:00.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:00.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.export 2026-02-20T21:07:00.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.export 2026-02-20T21:07:00.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:00.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export 2026-02-20T21:07:00.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.export 2026-02-20T21:07:00.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.export 2026-02-20T21:07:00.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:00.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:07:00.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:07:00.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export 2026-02-20T21:07:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:07:00.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:07:00.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:07:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:07:00.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:07:00.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:00.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster1 mirror test 2026-02-20T21:07:00.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:00.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:00.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:00.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-20T21:07:00.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:07:00.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-20T21:07:00.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:00.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:00.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:00.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-20T21:07:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:07:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:07:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:07:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:07:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:07:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:07:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:07:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:07:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:07:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:07:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:07:01.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:07:01.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:07:01.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:01.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:07:01.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:07:01.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:07:01.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:01.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:07:01.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:07:01.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:01.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:07:01.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:07:01.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:07:01.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:01.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:07:01.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:07:01.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:07:01.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:07:01.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:07:01.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:01.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:07:01.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:07:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:07:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:07:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:07:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:07:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:07:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:07:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:07:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:07:01.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:07:01.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:07:01.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:07:01.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:01.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:01.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:01.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:07:01.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:01.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:07:01.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:07:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:07:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:07:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:07:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:07:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:07:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:07:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:07:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:07:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:07:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:07:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:07:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:07:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:07:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:07:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:07:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:07:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:07:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:07:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:07:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:07:01.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:07:01.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:07:01.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:07:01.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:07:01.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:07:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:07:02.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:07:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:07:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:07:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:07:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:07:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:07:02.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:07:02.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:02.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:07:04.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:07:04.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:04.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:04.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:04.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:07:04.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:07:04.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:07:04.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:07:04.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:07:04.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:07:04.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:04.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:07:04.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:07:04.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:07:04.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:04.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:07:04.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:07:04.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:07:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:07:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:07:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:07:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:07:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:07:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:07:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:07:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:07:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:07:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:07:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:07:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:07:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:07:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:07:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:07:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:07:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:07:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:07:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:07:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:07:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:07:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:07:04.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:07:04.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:04.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:04.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:04.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:07:04.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:04.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:07:04.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:07:04.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:04.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:07:04.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:07:04.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:04.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:07:04.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:07:04.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:07:04.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:04.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:07:04.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:07:04.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:07:04.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:07:04.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:07:04.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:04.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:07:04.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:07:04.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:07:04.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:07:04.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:07:04.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:07:04.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:07:04.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:07:04.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:04.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:04.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:04.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:07:04.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:07:04.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:07:04.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:07:04.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:07:04.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:07:04.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:07:04.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:04.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:04.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:04.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:07:04.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:07:04.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:07:04.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:07:04.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:07:04.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:07:04.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:04.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:07:04.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:04.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:04.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:04.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:04.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:04.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:04.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:04.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:07:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:07:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:07:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:07:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:07:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:07:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:07:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:07:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:03 2026-02-20T21:07:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:07:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:07:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:07:05.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:07:05.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:02' 2026-02-20T21:07:05.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:07:05.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:07:05.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:07:05.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:07:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:07:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:07:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:07:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:07:05.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:07:05.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:07:05.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:07:05.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:07:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:03' 2026-02-20T21:07:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:07:03' 2026-02-20T21:07:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:07:03' 2026-02-20T21:07:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:07:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:07:03cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image demoted2026-02-20 21:07:02' 2026-02-20T21:07:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:07:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:07:05.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:03' 2026-02-20T21:07:05.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:07:05.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:07:05.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:07:05.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:07:05.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:07:05.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:05.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:05.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:07:05.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:05.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:07:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:07:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:07:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:07:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:07:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:07:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:07:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:07:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:02 2026-02-20T21:07:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:07:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:07:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:07:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:07:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:03' 2026-02-20T21:07:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:07:06.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:07:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:07:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image demoted 2026-02-20T21:07:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:07:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:07:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:07:06.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:07:06.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:07:06.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:07:06.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:07:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:07:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:02' 2026-02-20T21:07:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:07:02' 2026-02-20T21:07:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:07:02' 2026-02-20T21:07:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:06.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:07:06.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image demoted45294534mirror.1trial1992026-02-20 21:07:02cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:07:03' 2026-02-20T21:07:06.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:07:06.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:07:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:02' 2026-02-20T21:07:06.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:07:06.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:07:06.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:07:06.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:07:06.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:07:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster2 mirror test 2026-02-20T21:07:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:06.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:07:06.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-20T21:07:08.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:07:08.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:07:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:07:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:07:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:07:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:08.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:07:09.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:07:09.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:09.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:09.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:09.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:07:09.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:07:09.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:07:09.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:07:09.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:07:09.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:07:09.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:09.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:07:09.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:07:09.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:07:09.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:09.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:07:09.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:09.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:07:09.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:09.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:07:09.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:07:09.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:09.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:07:09.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:09.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:07:09.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:09.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:07:09.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:07:09.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:07:09.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:09.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:07:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:07:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:07:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:07:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:07:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:07:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:07:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:07:09.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:07:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:07:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:07:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:07:09.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:09.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:09.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:09.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:07:09.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:07:09.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:07:09.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:07:09.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:07:09.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:07:09.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:07:09.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:07:09.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:09.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test 100 2026-02-20T21:07:09.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:09.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:09.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:09.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:07:09.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:07:09.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:07:09.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:07:09.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:07:09.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:07:09.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:07:09.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 4545.52 bytes/sec: 18 MiB/s 2026-02-20T21:07:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:07:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:07:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:07:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:07:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:07:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:07:09.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:07:09.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:09.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:09.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:09.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:07:13.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 30 2026-02-20T21:07:13.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:07:13.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:07:13.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:13.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:07:13.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:07:13.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:13.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:13.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:07:13.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:13.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:07:13.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:07:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:07:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:13.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:07:13.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:07:13.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:07:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:07:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:07:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:07:13.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=30 2026-02-20T21:07:13.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:07:13.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:13.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:07:13.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:07:13.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:13.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:13.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:13.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:07:13.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:07:13.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:07:13.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:07:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=30 2026-02-20T21:07:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 30 = 30 2026-02-20T21:07:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-20T21:07:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:07:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:07:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:07:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:13.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:07:14.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:07:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:07:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:07:14.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:07:14.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:07:14.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:07:14.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621627,"replay_state":"idle"} 2026-02-20T21:07:14.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:07:14.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:08 2026-02-20T21:07:14.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:07:14.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:07:14.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:07:14.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:07:14.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:11' 2026-02-20T21:07:14.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:07:14.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:07:14.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:07:14.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621627,"replay_state":"idle"} 2026-02-20T21:07:14.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:07:14.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:07:14.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:07:14.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:07:14.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:07:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:07:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:07:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:07:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:08' 2026-02-20T21:07:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:07:08' 2026-02-20T21:07:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:14.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:07:14.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:07:08' 2026-02-20T21:07:14.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:14.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:14.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:07:14.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621627,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:07:08cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:07:11' 2026-02-20T21:07:14.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:07:14.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:07:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:08' 2026-02-20T21:07:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:07:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:07:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:07:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:07:14.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:07:14.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:14.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:14.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-20T21:07:14.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:14.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:14.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:14.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:07:14.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:14.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:14.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:14.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:07:15.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-20T21:07:15.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:15.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:15.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:15.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:07:15.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:15.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:15.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:15.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:15.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:07:15.443 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:07:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:07:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:07:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:07:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:07:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:11 2026-02-20T21:07:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:07:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:07:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:07:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621627,"replay_state":"idle"} 2026-02-20T21:07:15.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:08 2026-02-20T21:07:15.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:07:15.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 30 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.0e102c95-e0f7-4eb4-913b-ce015022fd9c (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:07:15.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:07:15.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:07:15.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:07:15.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:07:15.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:07:15.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:07:15.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:07:15.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:07:15.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:07:15.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:07:15.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:07:15.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:07:15.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:11' 2026-02-20T21:07:15.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-20 21:07:11' 2026-02-20T21:07:15.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:15.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:15.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:15.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:07:15.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:07:11' 2026-02-20T21:07:15.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:15.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:15.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:07:15.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:07:11cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621627,"replay_state":"idle"}2026-02-20 21:07:08' 2026-02-20T21:07:15.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:07:15.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:07:15.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:11' 2026-02-20T21:07:15.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:07:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:07:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:07:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:07:15.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:07:15.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:15.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:15.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror mirror test 2026-02-20T21:07:15.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:07:15.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:07:15.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:15.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:07:15.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:07:15.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:15.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.export 2026-02-20T21:07:15.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.export 2026-02-20T21:07:15.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:15.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export 2026-02-20T21:07:15.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.export 2026-02-20T21:07:15.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.export 2026-02-20T21:07:15.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:15.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:07:15.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:07:15.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export 2026-02-20T21:07:15.575 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:07:15.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:07:15.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:07:15.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:07:15.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:07:15.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:15.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: failover / failback loop' 2026-02-20T21:07:15.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:07:15.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:07:15.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:07:15 TEST: failover / failback loop 2026-02-20T21:07:15.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:07:15 TEST: failover / failback loop 2026-02-20T21:07:15.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 1 20 2026-02-20T21:07:15.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:07:15.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test 2026-02-20T21:07:15.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:15.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:15.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:15.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-20T21:07:16.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:07:16.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:07:16.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:16.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:16.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:16.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:07:16.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:16.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:16.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:16.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:07:16.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:07:16.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:16.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:07:17.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:07:17.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:17.512 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:17.512 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:17.512 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:07:17.512 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:07:17.512 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:07:17.512 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:07:17.512 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:07:17.512 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:17.512 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:07:17.512 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:07:17.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:07:17.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:07:17.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:17.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:07:17.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:17.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:07:17.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:17.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:07:17.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:07:17.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:17.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:07:17.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:17.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:07:17.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:17.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:07:17.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:07:17.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:07:17.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:17.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:07:17.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:07:17.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:07:17.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:17.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:17.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:07:17.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:07:17.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:07:17.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:17.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:07:17.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:17.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:17.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:07:17.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:17.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:17.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:07:17.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:07:17.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:07:17.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:07:17.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:17.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:17.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:17.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:17.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:07:17.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:07:17.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:07:17.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:07:17.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:07:17.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:07:17.575 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:07:17.575 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:07:17.575 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:17.575 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:07:19.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:07:19.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:19.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:19.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:19.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:07:19.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:07:19.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:07:19.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:07:19.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:07:19.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:19.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:07:19.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:07:19.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:07:19.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:19.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:07:19.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:07:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:07:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:07:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:07:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:07:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:07:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:07:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:07:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:07:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:07:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:07:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:07:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:07:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:07:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:07:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:07:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:07:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:07:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:07:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:07:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:07:19.585 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:19.593 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:19.593 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:19.593 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:19.593 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:07:19.641 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:07:19.642 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:07:19.642 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:07:19.642 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:07:19.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:07:19.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:19.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:07:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:19.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:07:20.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:07:20.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:20.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:20.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:20.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:20.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:20.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:20.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:20.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:07:20.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:07:20.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:07:20.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:07:20.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:07:20.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:07:20.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:18 2026-02-20T21:07:20.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:07:20.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:07:20.678 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:07:20.678 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:07:20.678 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:16' 2026-02-20T21:07:20.678 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:07:20.679 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:07:20.679 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:07:20.680 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:07:20.680 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:07:20.680 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:07:20.681 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:07:20.681 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:07:20.681 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:07:20.682 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:07:20.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:07:20.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:07:20.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:18' 2026-02-20T21:07:20.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:07:18' 2026-02-20T21:07:20.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:20.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:20.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:20.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:20.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:07:18' 2026-02-20T21:07:20.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:20.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:20.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:07:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:07:18cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:07:16' 2026-02-20T21:07:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:07:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:07:20.718 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:18' 2026-02-20T21:07:20.718 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:07:20.720 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:07:20.720 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:07:20.720 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:07:20.721 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:07:20.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:20.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:20.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:07:20.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:20.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:20.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:20.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:07:21.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:07:21.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:21.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:21.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:21.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:21.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:21.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:21.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:21.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:21.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:07:21.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:07:21.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:07:21.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:07:21.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:07:21.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:07:21.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:16 2026-02-20T21:07:21.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:07:21.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:07:21.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:07:21.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:07:21.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:18' 2026-02-20T21:07:21.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:07:21.756 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:07:21.756 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:07:21.756 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:07:21.757 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:07:21.757 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:07:21.757 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:07:21.757 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:07:21.758 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:07:21.758 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:07:21.758 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:07:21.759 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:07:21.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:16' 2026-02-20T21:07:21.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:07:16' 2026-02-20T21:07:21.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:21.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:21.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:21.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:21.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:07:16' 2026-02-20T21:07:21.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:21.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:21.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:07:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:07:21cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:07:18' 2026-02-20T21:07:21.792 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:07:21.792 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:07:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:21' 2026-02-20T21:07:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:07:21.795 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:07:21.795 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:07:21.795 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:07:21.796 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:07:21.797 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:21.797 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:21.797 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test 2026-02-20T21:07:21.797 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:21.797 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:21.798 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:21.798 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:07:21.798 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-20T21:07:22.853 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:07:22.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-20T21:07:22.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:22.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:22.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:22.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-20T21:07:22.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:22.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:22.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:22.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:07:22.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:07:22.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:22.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:07:23.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:07:23.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:23.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:23.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:23.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:07:23.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:07:23.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:07:23.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:07:23.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:07:23.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:23.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:07:23.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:07:23.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:07:23.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:07:23.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:23.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:07:23.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:07:23.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:07:23.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:23.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:07:23.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:07:23.866 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:23.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:07:23.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:07:23.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:07:23.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:23.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:07:23.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:07:23.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:07:23.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:07:23.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:07:23.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:23.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:07:23.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:07:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:07:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:07:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:07:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:07:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:07:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:07:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:07:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:07:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:07:23.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:07:23.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:07:23.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:23.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:23.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:23.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:07:23.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:07:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:07:23.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:23.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:07:23.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:07:23.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:23.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:07:23.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:07:23.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:07:23.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:23.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:07:23.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:07:23.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:07:23.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:07:23.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:07:23.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:23.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:07:23.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:07:23.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:07:23.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:07:23.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:07:23.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:07:23.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:07:23.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:07:23.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:23.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:23.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:23.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:07:23.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:07:23.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:07:23.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:07:23.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:07:23.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:07:23.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:07:23.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:23.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:23.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:23.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:07:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:07:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:07:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:07:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:07:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:07:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:23.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:07:25.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:07:25.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:25.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:25.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:07:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:07:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:07:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:07:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:07:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:07:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:07:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:07:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:07:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:07:25.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:07:25.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:07:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:07:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:07:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:07:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:07:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:07:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:07:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:07:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:07:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:07:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:07:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:07:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:07:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:07:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:07:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:07:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:07:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:07:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:07:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:07:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:07:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:07:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:07:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:07:26.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:26.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:26.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:26.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:07:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:07:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:07:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:07:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:07:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:07:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:07:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:07:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:07:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:07:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:07:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:07:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:07:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:07:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:07:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:07:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:07:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:07:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:07:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:07:26.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:07:26.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:26.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:26.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:26.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:07:26.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:07:26.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:07:26.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:07:26.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:07:26.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:07:26.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:07:26.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:26.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:26.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:26.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:07:26.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:07:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:07:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:07:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:07:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:07:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:07:30.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:07:30.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:30.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:30.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:30.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:07:30.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:07:30.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:07:30.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:07:30.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:07:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:07:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:07:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:07:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:07:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:07:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:07:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:07:30.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:30.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:07:30.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:07:30.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:30.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:07:30.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:07:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:07:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:07:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:07:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:07:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:07:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:07:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:07:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:07:30.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:07:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:07:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:07:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:07:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:07:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:07:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:07:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:07:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:07:30.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:07:30.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:07:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:07:30.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:30.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:07:30.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:07:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:07:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:07:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:07:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:07:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:07:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:07:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:07:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:07:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:07:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:07:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:07:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:07:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:07:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:07:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:07:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:07:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:07:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:07:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:07:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:07:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:07:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:07:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:07:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:07:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:07:30.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:30.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:30.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:30.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:07:30.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:07:30.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:07:30.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:07:30.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:07:30.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:07:30.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:07:30.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:07:30.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:30.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:07:30.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:07:30.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:30.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:07:30.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:07:30.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:30.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:07:30.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:07:30.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-20T21:07:30.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:30.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:30.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:30.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-20T21:07:33.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 33 2026-02-20T21:07:33.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:07:33.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:07:33.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:33.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:07:33.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:07:33.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:33.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:07:33.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:07:33.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:33.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:07:33.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:33.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:07:33.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:33.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:07:33.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:07:33.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:33.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:33.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:33.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:07:33.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:07:33.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:07:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:07:33.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=33 2026-02-20T21:07:33.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:07:33.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:33.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:07:34.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:07:34.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:34.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:34.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:34.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:07:34.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:07:34.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:07:34.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:07:34.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=33 2026-02-20T21:07:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 33 = 33 2026-02-20T21:07:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:07:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:07:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:07:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:07:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:07:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:35.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:35.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:07:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:07:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:07:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:07:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:07:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:07:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:33 2026-02-20T21:07:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:07:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:07:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:07:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621641,"replay_state":"idle"} 2026-02-20T21:07:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:26 2026-02-20T21:07:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:07:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 33 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.5e52802e-d9f8-4b41-bfff-0292a4bb90de (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:07:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:07:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:07:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:07:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:07:35.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:07:35.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:07:35.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:07:35.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:07:35.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:07:35.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:07:35.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:07:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:07:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:33' 2026-02-20T21:07:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:07:33' 2026-02-20T21:07:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:07:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:07:33' 2026-02-20T21:07:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:07:35.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary44044409mirror.0trial1992026-02-20 21:07:33cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621641,"replay_state":"idle"}2026-02-20 21:07:26' 2026-02-20T21:07:35.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:07:35.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:07:35.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:33' 2026-02-20T21:07:35.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:07:35.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:07:35.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:07:35.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:07:35.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:07:35.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:35.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:35.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-20T21:07:35.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:35.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:35.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:35.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:07:35.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:35.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:35.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:35.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:07:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-20T21:07:36.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:36.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:36.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:36.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:07:36.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:36.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:36.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:36.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:36.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:07:36.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:07:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:07:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:07:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621641,"replay_state":"idle"} 2026-02-20T21:07:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:07:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:26 2026-02-20T21:07:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:07:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:07:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:07:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:07:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:33' 2026-02-20T21:07:36.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:07:36.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:07:36.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:07:36.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621641,"replay_state":"idle"} 2026-02-20T21:07:36.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:07:36.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:07:36.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:07:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:07:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:07:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:07:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:07:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:07:36.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:26' 2026-02-20T21:07:36.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-20 21:07:26' 2026-02-20T21:07:36.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:36.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:36.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:36.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:07:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:07:26' 2026-02-20T21:07:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:07:36.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621641,"replay_state":"idle"}45294534mirror.1trial1992026-02-20 21:07:26cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:07:33' 2026-02-20T21:07:36.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:07:36.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:07:36.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:26' 2026-02-20T21:07:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:07:36.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:07:36.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:07:36.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:07:36.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:07:36.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:36.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:36.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster1 mirror test 2026-02-20T21:07:36.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:36.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:36.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:36.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-20T21:07:36.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:07:36.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-20T21:07:36.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:36.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:36.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:36.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-20T21:07:36.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:36.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:36.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:36.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:07:36.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:07:36.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:36.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:07:37.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:07:37.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:37.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:37.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:37.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:07:37.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:07:37.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:07:37.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:07:37.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:07:37.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:37.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:07:37.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:07:37.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:07:37.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:37.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:07:37.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:07:37.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:07:37.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:07:37.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:37.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:07:37.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:07:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:07:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:07:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:07:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:07:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:07:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:07:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:07:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:07:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:07:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:07:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:07:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:07:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:07:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:07:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:07:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:07:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:07:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:07:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:07:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:07:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:07:37.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:37.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:37.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:37.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:07:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:07:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:07:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:07:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:07:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:07:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:07:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:07:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:07:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:07:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:07:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:07:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:07:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:07:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:07:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:07:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:07:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:07:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:07:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:07:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:07:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:07:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:07:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:07:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:07:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:07:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:07:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:07:37.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:07:38.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:07:38.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:07:38.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:07:38.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:07:38.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:07:38.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:07:38.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:07:38.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:38.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:07:40.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:07:40.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:40.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:40.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:40.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:07:40.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:07:40.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:07:40.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:07:40.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:07:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:07:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:07:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:07:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:07:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:07:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:07:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:07:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:07:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:07:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:07:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:07:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:07:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:07:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:07:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:07:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:07:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:07:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:07:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:07:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:07:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:07:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:07:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:07:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:07:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:40.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:07:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:07:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:07:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:07:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:07:40.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:07:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:40.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:07:40.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:40.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:07:40.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:07:40.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:40.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:07:40.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:07:40.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:40.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:07:40.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:07:40.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:07:40.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:40.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:07:40.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:07:40.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:07:40.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:07:40.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:07:40.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:40.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:07:40.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:07:40.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:07:40.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:07:40.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:07:40.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:07:40.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:07:40.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:07:40.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:40.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:40.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:40.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:07:40.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:07:40.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:07:40.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:07:40.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:07:40.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:07:40.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:07:40.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:40.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:40.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:40.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:07:40.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:07:40.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:07:40.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:07:40.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:07:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:07:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:07:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:07:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:07:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:07:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:07:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:07:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:07:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:07:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:07:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:38 2026-02-20T21:07:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:07:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:07:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:07:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:07:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:38' 2026-02-20T21:07:41.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:07:41.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:07:41.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:07:41.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:07:41.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:07:41.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:07:41.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:07:41.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:07:41.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:07:41.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:07:41.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:07:41.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:07:41.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:38' 2026-02-20T21:07:41.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:07:38' 2026-02-20T21:07:41.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:41.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:41.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:41.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:41.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:07:38' 2026-02-20T21:07:41.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:41.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:41.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:07:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:07:38cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image demoted2026-02-20 21:07:38' 2026-02-20T21:07:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:07:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:07:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:38' 2026-02-20T21:07:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:07:41.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:07:41.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:07:41.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:07:41.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:07:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:07:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:07:42.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:07:42.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:42.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:42.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:42.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:42.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:42.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:42.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:42.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:42.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:07:42.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:07:42.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:07:42.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:07:42.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:07:42.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:07:42.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:41 2026-02-20T21:07:42.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:07:42.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:07:42.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:07:42.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:07:42.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:38' 2026-02-20T21:07:42.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:07:42.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:07:42.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:07:42.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:07:42.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:07:42.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:07:42.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:07:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:07:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:07:42.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:07:42.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:07:42.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:07:42.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:41' 2026-02-20T21:07:42.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:07:41' 2026-02-20T21:07:42.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:42.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:42.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:42.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:42.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:07:41' 2026-02-20T21:07:42.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:42.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:42.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:07:42.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:07:41cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:07:38' 2026-02-20T21:07:42.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:07:42.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:07:42.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:41' 2026-02-20T21:07:42.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:07:42.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:07:42.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:07:42.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:07:42.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:07:42.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:42.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:42.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster2 mirror test 2026-02-20T21:07:42.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:42.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:42.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:42.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:07:42.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-20T21:07:44.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:07:44.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:07:44.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:44.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:44.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:44.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:07:44.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:44.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:44.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:44.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:07:44.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:07:44.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:44.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:07:45.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:07:45.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:45.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:45.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:45.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:07:45.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:07:45.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:07:45.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:07:45.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:07:45.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:45.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:07:45.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:07:45.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:07:45.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:45.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:07:45.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:07:45.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:45.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:07:45.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:45.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:07:45.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:07:45.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:07:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:07:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:07:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:07:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:07:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:07:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:07:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:07:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:07:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:07:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:07:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:07:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:07:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:07:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:07:45.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:07:45.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:07:45.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:45.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:45.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:45.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:45.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:07:45.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:07:45.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:07:45.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:07:45.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:07:45.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:07:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:07:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:07:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:07:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:07:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:07:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:07:45.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:45.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:07:45.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:07:45.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:07:45.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:45.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:45.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:45.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:07:49.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 36 2026-02-20T21:07:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:07:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:07:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:07:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:07:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:49.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:07:49.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:49.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:07:49.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:07:49.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:07:49.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:49.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:07:49.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:07:49.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:49.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:49.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:49.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:07:49.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:07:49.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:07:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:07:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=36 2026-02-20T21:07:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:07:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:07:49.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:07:49.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:49.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:49.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:49.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:07:49.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:07:49.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:07:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:07:49.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=36 2026-02-20T21:07:49.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 36 = 36 2026-02-20T21:07:49.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:49.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-20T21:07:49.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:49.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:49.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:07:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:07:50.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-20T21:07:50.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:07:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:50.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:07:50.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:07:50.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:07:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:07:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:07:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:07:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:46 2026-02-20T21:07:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:07:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:07:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:07:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621663,"replay_state":"idle"} 2026-02-20T21:07:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:43 2026-02-20T21:07:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:07:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 36 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.7d03b7f9-7afb-4e65-96cf-957c4c0d5be4 (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:07:50.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:07:50.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:07:50.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:07:50.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:07:50.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:07:50.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:07:50.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:07:50.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:07:50.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:07:50.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:07:50.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:07:50.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:07:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:46' 2026-02-20T21:07:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-20 21:07:46' 2026-02-20T21:07:50.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:50.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:50.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:50.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:07:50.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:07:46' 2026-02-20T21:07:50.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:50.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:50.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:07:50.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:07:46cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621663,"replay_state":"idle"}2026-02-20 21:07:43' 2026-02-20T21:07:50.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:07:50.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:07:50.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:46' 2026-02-20T21:07:50.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:07:50.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:07:50.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:07:50.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:07:50.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:07:50.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:50.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:50.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:07:50.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:50.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:50.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:50.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:07:50.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:50.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:50.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:50.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:07:51.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:07:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:07:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:51.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:07:51.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:07:51.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:07:51.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:07:51.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621663,"replay_state":"idle"} 2026-02-20T21:07:51.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:07:51.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:43 2026-02-20T21:07:51.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:07:51.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:07:51.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:07:51.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:07:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:46' 2026-02-20T21:07:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:07:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:07:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:07:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621663,"replay_state":"idle"} 2026-02-20T21:07:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:07:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:07:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:07:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:07:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:07:51.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:07:51.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:07:51.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:07:51.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:43' 2026-02-20T21:07:51.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:07:43' 2026-02-20T21:07:51.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:51.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:51.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:51.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:07:51.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:07:43' 2026-02-20T21:07:51.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:51.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:51.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:07:51.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621663,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:07:43cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:07:46' 2026-02-20T21:07:51.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:07:51.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:07:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:43' 2026-02-20T21:07:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:07:51.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:07:51.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:07:51.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:07:51.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:07:51.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:51.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:51.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:07:51.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test 2026-02-20T21:07:51.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:51.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:51.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:51.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-20T21:07:52.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:07:52.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:07:52.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:52.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:52.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:52.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:07:52.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:52.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:52.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:52.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:07:52.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:07:52.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:52.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:07:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:07:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:07:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:07:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:07:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:07:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:07:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:07:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:07:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:07:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:07:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:53.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:07:53.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:53.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:07:53.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:53.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:07:53.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:07:53.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:53.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:07:53.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:53.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:07:53.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:53.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:07:53.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:07:53.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:07:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:07:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:07:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:07:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:07:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:07:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:07:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:07:53.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:53.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:53.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:07:53.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:53.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:53.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:07:53.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:07:53.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:07:53.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:07:53.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:53.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:53.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:53.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:07:53.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:07:53.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:07:53.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:07:53.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:07:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:07:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:07:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:07:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:07:55.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:07:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:07:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:07:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:07:55.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:07:55.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:07:55.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:07:55.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:55.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:07:55.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:07:55.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:07:55.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:55.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:07:55.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:55.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:07:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:07:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:07:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:07:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:07:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:07:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:07:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:07:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:07:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:07:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:07:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:07:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:07:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:07:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:07:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:07:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:07:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:07:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:07:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:07:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:55.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:55.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:55.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:55.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:07:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:07:55.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:07:55.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:07:55.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:07:55.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:07:55.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:55.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:07:55.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:55.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:55.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:55.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:55.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:55.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:55.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:55.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:07:56.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:07:56.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:56.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:56.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:56.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:56.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:56.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:56.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:56.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:56.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:07:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:07:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:07:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:07:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:07:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:07:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:53 2026-02-20T21:07:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:07:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:07:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:07:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:07:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:51' 2026-02-20T21:07:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:07:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:07:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:07:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image demoted 2026-02-20T21:07:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:07:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:07:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:07:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:07:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:07:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:07:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:07:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:07:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:53' 2026-02-20T21:07:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:07:53' 2026-02-20T21:07:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:56.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:56.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:07:53' 2026-02-20T21:07:56.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:56.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:56.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:07:56.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image demoted44044409mirror.0trial1992026-02-20 21:07:53cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:07:51' 2026-02-20T21:07:56.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:07:56.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:07:56.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:53' 2026-02-20T21:07:56.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:07:56.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:07:56.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:07:56.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:07:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:07:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:07:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:07:57.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:07:57.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:57.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:57.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:57.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:57.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:07:57.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:07:57.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:57.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:57.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:07:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:07:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:07:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:07:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:07:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:07:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:56 2026-02-20T21:07:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:07:57.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:07:57.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:07:57.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:07:57.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:07:53' 2026-02-20T21:07:57.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:07:57.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:07:57.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:07:57.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:07:57.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:07:57.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:07:57.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:07:57.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:07:57.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:07:57.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:07:57.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:07:57.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:07:57.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:56' 2026-02-20T21:07:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:07:56' 2026-02-20T21:07:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:07:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:07:56' 2026-02-20T21:07:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:07:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:07:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:07:57.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:07:56cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image demoted2026-02-20 21:07:53' 2026-02-20T21:07:57.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:07:57.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:07:57.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:07:56' 2026-02-20T21:07:57.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:07:57.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:07:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:07:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:07:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:07:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:07:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test 2026-02-20T21:07:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:07:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:07:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-20T21:07:58.855 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:07:58.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-20T21:07:58.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:58.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:58.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:58.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-20T21:07:58.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:58.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:58.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:58.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:07:58.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:07:58.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:58.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:07:59.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:07:59.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:07:59.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:07:59.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:07:59.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:07:59.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:07:59.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:07:59.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:07:59.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:07:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:07:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:07:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:07:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:07:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:07:59.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:07:59.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:07:59.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:59.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:07:59.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:07:59.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:59.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:07:59.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:07:59.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:07:59.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:59.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:07:59.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:07:59.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:07:59.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:07:59.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:07:59.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:59.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:07:59.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:59.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:07:59.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:07:59.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:07:59.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:07:59.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:07:59.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:07:59.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:59.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:59.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:59.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:07:59.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:07:59.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:07:59.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:07:59.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:07:59.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:07:59.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:07:59.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:59.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:59.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:07:59.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:07:59.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:59.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:07:59.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:07:59.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:59.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:07:59.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:07:59.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:07:59.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:07:59.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:07:59.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:07:59.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:59.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:07:59.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:07:59.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:07:59.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:07:59.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:07:59.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:59.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:07:59.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:07:59.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:07:59.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:07:59.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:07:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:07:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:07:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:07:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:07:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:07:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:07:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:07:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:07:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:07:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:07:59.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:07:59.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:59.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:59.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:07:59.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:07:59.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:07:59.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:07:59.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:07:59.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:07:59.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:07:59.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:59.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:08:01.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:08:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:08:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:08:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:08:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:08:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:08:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:08:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:08:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:08:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:08:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:08:01.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:08:01.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:08:01.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:01.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:08:01.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:08:01.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:01.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:08:01.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:08:01.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:08:01.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:01.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:08:01.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:08:01.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:08:01.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:08:01.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:08:01.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:01.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:08:01.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:08:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:08:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:08:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:08:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:08:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:08:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:08:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:08:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:08:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:08:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:08:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:08:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:01.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:01.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:01.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:08:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:02.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:08:02.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:08:02.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:02.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:08:02.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:08:02.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:02.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:08:02.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:08:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:08:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:08:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:08:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:08:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:08:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:08:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:08:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:08:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:08:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:08:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:08:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:08:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:08:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:08:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:08:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:08:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:08:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:08:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:08:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:08:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:08:02.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:02.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:02.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:02.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:08:02.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:08:02.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:08:02.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:08:02.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:08:02.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:08:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:08:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:08:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:08:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:08:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:02.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:08:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:08:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:08:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:08:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-20T21:08:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-20T21:08:05.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 39 2026-02-20T21:08:05.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:08:05.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:08:05.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:05.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:08:05.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:08:05.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:05.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:08:05.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:08:05.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:08:05.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:08:05.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:08:05.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:08:05.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:08:05.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:08:05.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:08:05.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:05.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:05.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:05.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:08:05.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:08:05.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:08:05.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:08:05.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=39 2026-02-20T21:08:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:08:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:08:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:08:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:08:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:08:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:08:06.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:08:06.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:08:06.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=39 2026-02-20T21:08:06.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 39 = 39 2026-02-20T21:08:06.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:06.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:08:06.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:06.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:06.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:06.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:08:06.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:06.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:06.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:06.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:08:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:08:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:08:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:08:07.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:08:07.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:08:07.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:08:07.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:08:07.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:08:07.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:03 2026-02-20T21:08:07.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:08:07.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:08:07.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:08:07.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621677,"replay_state":"idle"} 2026-02-20T21:08:07.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:01 2026-02-20T21:08:07.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:08:07.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 39 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.3f94544d-3580-4e12-ae83-26cfea663443 (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:08:07.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:08:07.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:08:07.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:08:07.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:08:07.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:08:07.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:08:07.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:08:07.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:08:07.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:08:07.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:08:07.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:08:07.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:08:07.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:03' 2026-02-20T21:08:07.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:08:03' 2026-02-20T21:08:07.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:07.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:07.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:07.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:08:07.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:08:03' 2026-02-20T21:08:07.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:07.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:07.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:08:07.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary44044409mirror.0trial1992026-02-20 21:08:03cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621677,"replay_state":"idle"}2026-02-20 21:08:01' 2026-02-20T21:08:07.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:08:07.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:08:07.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:03' 2026-02-20T21:08:07.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:08:07.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:08:07.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:08:07.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:08:07.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:08:07.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:07.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:07.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-20T21:08:07.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:07.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:07.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:07.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:08:07.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:07.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:07.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:07.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:08:08.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-20T21:08:08.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:08.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:08.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:08.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:08:08.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:08.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:08.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:08.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:08.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:08:08.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:08:08.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:08:08.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:08:08.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621677,"replay_state":"idle"} 2026-02-20T21:08:08.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:08:08.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:01 2026-02-20T21:08:08.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:08:08.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:08:08.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:08:08.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:08:08.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:08' 2026-02-20T21:08:08.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:08:08.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:08:08.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:08:08.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621677,"replay_state":"idle"} 2026-02-20T21:08:08.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:08:08.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:08:08.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:08:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:08:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:08:08.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:08:08.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:08:08.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:08:08.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:01' 2026-02-20T21:08:08.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-20 21:08:01' 2026-02-20T21:08:08.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:08.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:08.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:08.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:08:08.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:08:01' 2026-02-20T21:08:08.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:08.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:08.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:08:08.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621677,"replay_state":"idle"}45294534mirror.1trial1992026-02-20 21:08:01cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:08:08' 2026-02-20T21:08:08.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:08:08.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:08:08.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:01' 2026-02-20T21:08:08.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:08:08.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:08:08.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:08:08.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:08:08.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:08:08.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:08.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:08.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster1 mirror test 2026-02-20T21:08:08.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:08.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:08.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:08.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-20T21:08:08.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:08:08.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-20T21:08:08.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:08.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:08.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:08.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-20T21:08:08.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:08.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:08.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:08.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:08:08.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:08:08.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:08.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:08:09.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:08:09.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:09.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:09.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:09.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:08:09.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:08:09.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:08:09.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:08:09.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:08:09.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:09.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:08:09.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:08:09.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:08:09.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:08:09.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:09.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:08:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:08:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:08:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:08:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:08:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:08:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:08:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:08:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:08:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:08:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:08:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:08:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:08:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:08:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:08:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:08:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:08:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:08:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:08:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:08:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:08:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:08:09.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:08:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:08:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:08:09.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:08:09.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:09.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:09.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:09.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:08:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:08:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:08:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:08:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:08:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:08:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:08:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:08:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:08:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:08:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:08:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:08:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:08:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:08:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:08:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:08:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:08:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:08:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:08:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:08:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:08:10.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:10.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:10.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:10.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:08:10.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:08:10.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:08:10.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:08:10.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:08:10.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:08:10.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:08:10.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:10.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:10.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:10.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:08:10.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:08:10.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:08:10.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:08:10.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:08:10.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:08:10.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:08:10.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:08:10.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:10.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:08:12.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:08:12.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:12.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:12.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:12.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:08:12.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:08:12.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:08:12.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:08:12.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:08:12.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:08:12.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:12.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:08:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:08:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:08:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:08:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:08:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:08:12.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:12.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:08:12.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:08:12.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:12.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:08:12.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:08:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:08:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:08:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:08:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:08:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:08:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:08:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:08:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:08:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:08:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:08:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:08:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:08:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:08:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:08:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:08:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:08:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:08:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:08:12.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:08:12.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:12.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:12.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:12.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:08:12.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:12.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:08:12.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:08:12.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:12.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:08:12.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:08:12.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:12.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:08:12.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:08:12.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:08:12.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:12.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:08:12.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:08:12.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:08:12.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:08:12.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:08:12.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:12.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:08:12.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:08:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:08:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:08:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:08:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:08:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:08:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:08:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:08:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:08:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:08:12.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:08:12.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:08:12.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:08:12.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:08:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:08:12.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:08:12.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:08:12.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:08:12.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:08:12.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:08:12.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:12.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:08:12.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:12.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:12.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:12.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:08:12.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:12.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:12.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:12.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:08:13.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:08:13.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:13.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:13.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:13.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:08:13.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:13.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:13.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:13.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:13.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:08:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:08:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:08:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:08:13.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:08:13.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:08:13.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:13 2026-02-20T21:08:13.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:08:13.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:08:13.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:08:13.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:08:13.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:11' 2026-02-20T21:08:13.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:08:13.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:08:13.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:08:13.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:08:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:08:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:08:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:08:13.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:08:13.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:08:13.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:08:13.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:08:13.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:08:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:13' 2026-02-20T21:08:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:08:13' 2026-02-20T21:08:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:08:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:08:13' 2026-02-20T21:08:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:08:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:08:13cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:08:11' 2026-02-20T21:08:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:08:13.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:08:13.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:13' 2026-02-20T21:08:13.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:08:13.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:08:13.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:08:13.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:08:13.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:08:13.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:13.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:13.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:08:13.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:13.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:13.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:13.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:08:13.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:13.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:13.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:13.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:08:14.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:08:14.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:14.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:14.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:14.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:08:14.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:14.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:14.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:14.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:14.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:08:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:08:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:08:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:08:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:08:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:08:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:11 2026-02-20T21:08:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:08:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:08:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:08:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:08:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:13' 2026-02-20T21:08:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:08:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:08:14.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:08:14.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:08:14.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:08:14.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:08:14.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:08:14.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:08:14.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:08:14.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:08:14.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:08:14.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:08:14.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:11' 2026-02-20T21:08:14.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:08:11' 2026-02-20T21:08:14.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:14.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:14.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:14.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:08:14.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:08:11' 2026-02-20T21:08:14.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:14.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:14.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:08:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:08:11cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:08:13' 2026-02-20T21:08:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:08:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:08:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:11' 2026-02-20T21:08:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:08:14.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:08:14.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:08:14.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:08:14.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:08:14.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:14.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:14.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster2 mirror test 2026-02-20T21:08:14.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:14.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:14.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:14.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:08:14.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-20T21:08:16.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:08:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:08:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:08:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:08:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:08:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:08:17.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:08:17.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:17.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:17.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:17.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:08:17.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:08:17.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:08:17.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:08:17.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:08:17.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:17.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:08:17.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:08:17.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:08:17.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:08:17.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:17.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:08:17.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:08:17.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:08:17.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:17.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:08:17.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:08:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:08:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:08:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:08:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:08:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:08:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:08:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:08:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:08:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:08:17.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:17.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:08:17.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:08:17.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:08:17.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:08:17.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:08:17.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:08:17.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:17.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:17.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:08:17.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:08:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:08:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:08:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:08:17.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:08:17.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:17.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:17.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:17.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:08:17.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:08:17.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:08:17.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:08:17.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:08:17.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:08:17.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:17.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:08:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:08:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:08:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:08:19.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:08:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:08:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:08:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:08:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:08:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:08:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:08:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:08:19.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:08:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:08:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:08:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:08:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:08:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:08:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:08:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:08:19.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:08:19.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:08:19.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:19.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:08:19.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:08:19.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:08:19.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:19.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:08:19.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:08:19.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:08:19.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:08:19.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:08:19.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:08:19.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:19.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:19.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:08:19.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:19.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:19.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:08:19.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:08:19.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:08:19.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:08:19.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:08:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:08:19.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:08:19.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:08:19.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:08:19.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:08:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:08:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:08:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:08:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:08:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:08:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:08:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:08:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:19.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:08:19.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:08:19.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:08:19.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:19.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:19.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:19.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:08:23.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 42 2026-02-20T21:08:23.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:08:23.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:08:23.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:23.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:08:23.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:08:23.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:23.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:08:23.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:08:23.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:08:23.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:08:23.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:08:23.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:08:23.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:08:23.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:08:23.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:08:23.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:23.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:23.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:23.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:08:23.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:08:23.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:08:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:08:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=42 2026-02-20T21:08:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:08:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:08:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:08:23.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:08:23.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:23.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:23.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:23.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:08:23.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:08:23.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:08:23.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:08:23.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=42 2026-02-20T21:08:23.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 42 = 42 2026-02-20T21:08:23.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:23.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-20T21:08:23.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:23.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:23.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:23.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:08:23.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:23.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:23.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:23.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:08:24.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-20T21:08:24.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:24.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:24.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:24.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:08:24.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:24.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:08:24.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:08:24.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:08:24.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:08:24.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:08:24.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:08:24.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:21 2026-02-20T21:08:24.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:08:24.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:08:24.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:08:24.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621695,"replay_state":"idle"} 2026-02-20T21:08:24.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:18 2026-02-20T21:08:24.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:08:24.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 42 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.36b821c7-d1bd-4348-99ec-cbef00ffd29f (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:08:24.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:08:24.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:08:24.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:08:24.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:08:24.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:08:24.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:08:24.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:08:24.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:08:24.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:08:24.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:08:24.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:08:24.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:08:24.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:21' 2026-02-20T21:08:24.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-20 21:08:21' 2026-02-20T21:08:24.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:24.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:24.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:24.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:08:24.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:08:21' 2026-02-20T21:08:24.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:24.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:24.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:08:24.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:08:21cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621695,"replay_state":"idle"}2026-02-20 21:08:18' 2026-02-20T21:08:24.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:08:24.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:08:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:21' 2026-02-20T21:08:24.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:08:24.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:08:24.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:08:24.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:08:24.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:08:24.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:24.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:24.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:08:24.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:24.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:24.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:24.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:08:24.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:24.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:24.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:24.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:08:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:08:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:25.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:25.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:25.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:08:25.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:25.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:25.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:25.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:25.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:08:25.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:08:25.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:08:25.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:08:25.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621695,"replay_state":"idle"} 2026-02-20T21:08:25.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:08:25.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:18 2026-02-20T21:08:25.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:08:25.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:08:25.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:08:25.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:08:25.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:21' 2026-02-20T21:08:25.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:08:25.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:08:25.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:08:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621695,"replay_state":"idle"} 2026-02-20T21:08:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:08:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:08:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:08:25.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:08:25.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:08:25.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:08:25.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:08:25.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:08:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:18' 2026-02-20T21:08:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:08:18' 2026-02-20T21:08:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:08:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:08:18' 2026-02-20T21:08:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:08:25.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621695,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:08:18cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:08:21' 2026-02-20T21:08:25.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:08:25.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:08:25.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:18' 2026-02-20T21:08:25.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:08:25.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:08:25.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:08:25.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:08:25.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:08:25.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:25.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:25.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:08:25.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test 2026-02-20T21:08:25.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:25.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:25.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:25.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-20T21:08:26.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:08:26.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:08:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:08:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:08:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:08:26.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:26.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:08:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:08:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:08:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:08:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:08:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:08:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:08:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:08:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:08:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:08:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:08:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:08:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:08:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:08:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:08:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:08:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:08:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:08:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:08:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:08:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:08:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:08:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:08:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:08:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:08:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:08:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:08:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:08:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:08:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:08:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:08:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:27.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:08:27.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:27.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:27.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:08:27.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:08:27.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:08:27.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:08:27.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:08:27.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:27.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:27.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:27.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:08:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:08:27.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:08:27.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:08:27.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:08:27.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:08:27.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:08:27.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:08:27.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:27.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:08:29.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:08:29.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:29.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:29.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:29.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:08:29.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:08:29.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:08:29.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:08:29.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:08:29.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:08:29.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:29.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:08:29.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:08:29.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:08:29.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:29.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:08:29.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:08:29.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:08:29.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:29.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:08:29.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:08:29.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:29.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:08:29.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:08:29.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:08:29.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:29.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:08:29.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:08:29.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:08:29.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:29.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:08:29.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:08:29.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:08:29.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:29.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:08:29.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:08:29.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:08:29.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:08:29.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:08:29.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:08:29.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:29.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:29.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:08:29.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:29.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:29.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:08:29.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:08:29.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:08:29.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:08:29.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:08:29.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:29.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:29.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:08:29.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:08:29.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:08:29.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:08:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:08:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:08:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:08:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:08:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:08:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:08:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:08:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:30.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:30.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:30.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:30.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:08:30.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:08:30.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:08:30.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:08:30.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:08:30.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:08:30.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:28 2026-02-20T21:08:30.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:08:30.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:08:30.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:08:30.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:08:30.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:26' 2026-02-20T21:08:30.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:08:30.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:08:30.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:08:30.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:08:30.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:08:30.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:08:30.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:08:30.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:08:30.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:08:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:08:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:08:30.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:08:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:28' 2026-02-20T21:08:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:08:28' 2026-02-20T21:08:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:08:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:08:28' 2026-02-20T21:08:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:30.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:08:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:08:28cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:08:26' 2026-02-20T21:08:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:08:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:08:30.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:28' 2026-02-20T21:08:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:08:30.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:08:30.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:08:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:08:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:08:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:08:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:08:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:08:31.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:08:31.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:31.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:31.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:31.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:08:31.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:31.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:31.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:31.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:31.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:08:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:08:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:08:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:08:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:08:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:08:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:26 2026-02-20T21:08:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:08:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:08:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:08:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:08:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:28' 2026-02-20T21:08:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:08:31.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:08:31.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:08:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:08:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:08:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:08:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:08:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:08:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:08:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:08:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:08:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:08:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:26' 2026-02-20T21:08:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:08:26' 2026-02-20T21:08:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:08:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:08:26' 2026-02-20T21:08:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:08:31.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:08:26cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:08:28' 2026-02-20T21:08:31.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:08:31.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:08:31.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:26' 2026-02-20T21:08:31.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:08:31.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:08:31.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:08:31.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:08:31.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:08:31.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:31.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:31.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test 2026-02-20T21:08:31.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:31.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:31.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:31.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:08:31.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-20T21:08:32.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:08:32.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-20T21:08:32.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:32.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:32.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:32.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-20T21:08:32.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:32.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:32.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:32.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:08:32.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:08:32.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:32.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:08:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:08:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:08:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:08:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:08:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:08:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:08:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:08:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:08:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:08:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:08:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:08:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:08:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:08:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:08:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:08:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:08:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:08:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:08:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:33.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:08:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:08:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:08:34.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:34.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:08:34.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:08:34.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:34.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:08:34.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:08:34.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:08:34.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:34.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:08:34.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:08:34.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:08:34.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:08:34.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:08:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:08:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:08:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:08:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:08:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:08:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:08:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:08:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:08:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:08:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:08:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:08:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:08:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:08:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:08:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:08:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:08:34.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:08:34.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:08:34.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:08:34.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:08:34.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:08:34.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:34.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:08:36.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:08:36.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:36.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:36.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:36.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:08:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:08:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:08:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:08:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:08:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:08:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:08:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:08:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:08:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:08:36.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:08:36.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:08:36.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:36.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:08:36.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:08:36.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:36.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:08:36.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:08:36.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:08:36.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:36.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:08:36.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:08:36.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:08:36.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:08:36.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:08:36.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:36.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:08:36.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:36.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:08:36.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:08:36.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:08:36.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:08:36.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:08:36.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:08:36.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:36.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:36.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:36.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:36.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:08:36.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:08:36.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:08:36.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:08:36.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:08:36.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:08:36.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:36.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:36.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:36.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:08:36.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:36.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:08:36.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:08:36.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:36.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:08:36.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:08:36.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:36.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:08:36.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:08:36.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:08:36.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:36.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:08:36.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:08:36.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:08:36.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:08:36.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:08:36.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:36.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:08:36.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:08:36.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:08:36.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:08:36.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:08:36.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:08:36.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:08:36.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:08:36.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:36.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:36.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:36.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:08:36.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:08:36.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:08:36.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:08:36.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:08:36.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:08:36.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:08:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:08:36.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:08:36.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:08:36.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:08:36.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:08:36.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:08:36.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:36.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:08:40.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:08:40.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:40.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:40.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:40.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:08:40.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:08:40.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:08:40.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:08:40.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:08:40.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:08:40.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:40.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:08:40.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:08:40.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:08:40.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:40.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:08:40.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:08:40.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:08:40.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:40.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:08:40.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:08:40.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:40.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:08:40.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:08:40.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:08:40.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:40.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:08:40.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:08:40.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:08:40.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:08:40.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:08:40.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:40.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:08:40.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:40.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:08:40.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:08:40.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:08:40.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:08:40.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:08:40.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:08:40.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:40.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:40.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:40.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:40.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:08:40.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:08:40.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:08:40.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:08:40.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:08:40.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:08:40.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:40.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:40.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:40.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:08:40.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:40.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:08:40.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:08:40.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:40.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:08:40.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:08:40.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:40.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:08:40.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:08:40.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:08:40.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:40.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:08:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:08:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:08:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:08:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:08:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:08:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:08:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:08:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:08:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:08:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:08:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:08:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:08:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:08:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:08:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:08:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:08:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:08:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:08:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:08:40.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:40.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:40.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:40.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:08:40.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:08:40.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:08:40.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:08:40.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:08:40.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:08:40.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:08:40.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:08:40.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:40.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:08:40.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:08:40.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:08:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:08:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:08:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:08:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-20T21:08:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-20T21:08:43.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 45 2026-02-20T21:08:43.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:08:43.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:08:43.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:43.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:08:43.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:08:43.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:43.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:08:43.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:08:43.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:08:43.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:08:43.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:08:43.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:08:43.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:08:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:08:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:08:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:08:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:08:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:08:43.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:08:43.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=45 2026-02-20T21:08:43.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:08:43.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:08:43.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:08:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:08:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:08:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:08:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:08:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:08:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=45 2026-02-20T21:08:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 45 = 45 2026-02-20T21:08:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:08:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:08:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:08:45.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:08:45.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:45.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:45.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:45.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:08:45.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:45.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:45.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:45.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:45.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:08:45.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:08:45.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:08:45.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:08:45.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:08:45.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:08:45.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:43 2026-02-20T21:08:45.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:08:45.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:08:45.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:08:45.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621711,"replay_state":"idle"} 2026-02-20T21:08:45.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:36 2026-02-20T21:08:45.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:08:45.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 45 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.e7cb27fd-23f5-4d9d-8f2c-3492d1b4eabf (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:08:45.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:08:45.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:08:45.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:08:45.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:08:45.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:08:45.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:08:45.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:08:45.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:08:45.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:08:45.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:08:45.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:08:45.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:08:45.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:43' 2026-02-20T21:08:45.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:08:43' 2026-02-20T21:08:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:08:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:08:43' 2026-02-20T21:08:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:45.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:08:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary44044409mirror.0trial1992026-02-20 21:08:43cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621711,"replay_state":"idle"}2026-02-20 21:08:36' 2026-02-20T21:08:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:08:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:08:45.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:43' 2026-02-20T21:08:45.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:08:45.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:08:45.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:08:45.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:08:45.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:08:45.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:45.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:45.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-20T21:08:45.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:45.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:45.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:45.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:08:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:08:46.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-20T21:08:46.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:46.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:46.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:46.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:08:46.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:46.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:46.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:46.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:46.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:08:46.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:08:46.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:08:46.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:08:46.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621711,"replay_state":"idle"} 2026-02-20T21:08:46.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:08:46.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:36 2026-02-20T21:08:46.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:08:46.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:08:46.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:08:46.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:08:46.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:43' 2026-02-20T21:08:46.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:08:46.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:08:46.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:08:46.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621711,"replay_state":"idle"} 2026-02-20T21:08:46.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:08:46.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:08:46.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:08:46.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:08:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:08:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:08:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:08:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:08:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:36' 2026-02-20T21:08:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-20 21:08:36' 2026-02-20T21:08:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:46.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:08:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:08:36' 2026-02-20T21:08:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:46.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:08:46.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621711,"replay_state":"idle"}45294534mirror.1trial1992026-02-20 21:08:36cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:08:43' 2026-02-20T21:08:46.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:08:46.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:08:46.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:36' 2026-02-20T21:08:46.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:08:46.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:08:46.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:08:46.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:08:46.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:08:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster1 mirror test 2026-02-20T21:08:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:46.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-20T21:08:46.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:08:46.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-20T21:08:46.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:46.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:46.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:46.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-20T21:08:46.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:46.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:46.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:46.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:08:46.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:08:46.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:46.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:08:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:08:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:47.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:08:47.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:08:47.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:08:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:08:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:08:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:08:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:08:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:08:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:08:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:08:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:08:47.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:08:47.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:47.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:08:47.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:08:47.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:47.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:08:47.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:08:47.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:08:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:08:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:08:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:08:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:08:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:08:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:08:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:08:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:08:47.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:08:47.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:08:47.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:08:47.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:08:47.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:47.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:47.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:47.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:47.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:08:47.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:08:47.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:08:47.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:08:47.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:08:47.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:08:47.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:47.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:47.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:47.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:08:47.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:47.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:08:47.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:08:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:47.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:08:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:08:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:08:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:08:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:08:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:08:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:08:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:08:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:08:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:08:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:47.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:08:47.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:08:47.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:08:47.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:08:47.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:08:47.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:08:47.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:08:47.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:08:47.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:47.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:47.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:47.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:08:47.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:08:47.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:08:47.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:08:47.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:08:47.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:08:47.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:08:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:08:47.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:08:47.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:08:47.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:08:47.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:08:47.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:08:47.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:08:47.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:08:47.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:47.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:08:49.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:08:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:08:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:08:50.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:08:50.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:08:50.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:08:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:08:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:08:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:08:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:08:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:08:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:08:50.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:08:50.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:08:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:08:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:08:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:08:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:08:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:08:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:08:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:08:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:08:50.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:08:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:08:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:08:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:08:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:08:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:08:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:08:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:08:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:50.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:08:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:08:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:08:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:08:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:08:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:08:50.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:50.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:50.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:08:50.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:08:50.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:08:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:08:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:08:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:08:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:08:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:08:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:08:50.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:08:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:08:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:08:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:08:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:08:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:08:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:08:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:08:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:08:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:08:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:08:50.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:08:50.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:08:50.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:50.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:50.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:08:50.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:08:50.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:08:50.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:08:50.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:08:50.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:08:50.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:08:50.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:08:50.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:50.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:50.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:08:50.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:08:50.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:08:50.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:08:50.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:08:50.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:08:50.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:08:50.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:50.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:08:50.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:50.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:50.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:50.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:08:50.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:50.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:50.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:50.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:08:51.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:08:51.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:08:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:08:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:08:51.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:08:51.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:08:51.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:08:51.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:08:51.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:48 2026-02-20T21:08:51.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:08:51.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:08:51.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:08:51.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:08:51.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:48' 2026-02-20T21:08:51.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:08:51.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:08:51.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:08:51.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:08:51.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:08:51.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:08:51.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:08:51.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:08:51.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:08:51.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:08:51.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:08:51.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:08:51.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:48' 2026-02-20T21:08:51.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:08:48' 2026-02-20T21:08:51.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:51.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:51.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:51.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:08:51.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:08:48' 2026-02-20T21:08:51.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:51.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:51.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:08:51.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:08:48cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image demoted2026-02-20 21:08:48' 2026-02-20T21:08:51.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:08:51.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:08:51.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:48' 2026-02-20T21:08:51.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:08:51.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:08:51.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:08:51.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:08:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:08:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:08:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:08:51.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:51.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:51.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:51.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:08:52.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:08:52.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:52.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:52.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:52.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:08:52.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:52.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:52.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:52.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:52.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:08:52.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:08:52.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:08:52.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:08:52.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:08:52.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:08:52.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:51 2026-02-20T21:08:52.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:08:52.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:08:52.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:08:52.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:08:52.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:48' 2026-02-20T21:08:52.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:08:52.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:08:52.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:08:52.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:08:52.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:08:52.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:08:52.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:08:52.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:08:52.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:08:52.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:08:52.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:08:52.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:08:52.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:51' 2026-02-20T21:08:52.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:08:51' 2026-02-20T21:08:52.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:52.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:52.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:52.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:08:52.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:08:51' 2026-02-20T21:08:52.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:52.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:52.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:08:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:08:51cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:08:48' 2026-02-20T21:08:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:08:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:08:52.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:51' 2026-02-20T21:08:52.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:08:52.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:08:52.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:08:52.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:08:52.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:08:52.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:52.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster2 mirror test 2026-02-20T21:08:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:08:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-20T21:08:53.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:08:53.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:08:53.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:53.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:53.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:08:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:08:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:08:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:08:54.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:08:54.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:54.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:54.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:54.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:08:54.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:08:54.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:08:54.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:08:54.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:08:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:08:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:08:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:08:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:08:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:54.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:08:54.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:08:54.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:08:54.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:54.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:08:54.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:08:54.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:08:54.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:08:54.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:08:54.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:08:54.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:54.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:08:54.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:08:54.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:08:54.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:54.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:08:54.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:08:54.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:08:54.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:54.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:08:54.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:08:54.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:08:54.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:08:54.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:08:54.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:08:54.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:54.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:54.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:08:54.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:08:54.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:54.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:08:54.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:08:54.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:08:54.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:08:54.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:08:54.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:54.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:54.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:54.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:08:54.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:08:54.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:08:54.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:08:54.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:08:54.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:08:54.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:08:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:08:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:08:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:08:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:08:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:08:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:08:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:08:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:08:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:54.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:54.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:54.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:08:58.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 48 2026-02-20T21:08:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:08:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:08:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:08:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:08:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:08:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:08:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:08:58.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:08:58.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:08:58.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:08:58.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:08:58.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:08:58.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:08:58.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:58.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:58.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:58.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:08:58.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:08:58.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:08:58.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:08:58.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=48 2026-02-20T21:08:58.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:08:58.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:08:58.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:08:58.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:08:58.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:58.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:58.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:58.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:08:58.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:08:58.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:08:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:08:58.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=48 2026-02-20T21:08:58.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 48 = 48 2026-02-20T21:08:58.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:58.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-20T21:08:58.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:58.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:58.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:58.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:08:58.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:58.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:58.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:58.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:08:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-20T21:08:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:08:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:08:59.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:08:59.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:08:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:08:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:08:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:08:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:56 2026-02-20T21:08:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:08:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:08:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:08:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621735,"remote_snapshot_timestamp":1771621735,"replay_state":"idle"} 2026-02-20T21:08:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:58 2026-02-20T21:08:59.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:08:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 48 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.9f8d02eb-002d-4a4a-aa64-b843843995ab (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:08:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:08:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:08:59.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:08:59.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:08:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:08:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:08:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:08:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:08:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:08:59.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:08:59.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:08:59.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:08:59.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:56' 2026-02-20T21:08:59.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-20 21:08:56' 2026-02-20T21:08:59.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:08:59.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:59.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:59.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:08:59.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:08:56' 2026-02-20T21:08:59.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:08:59.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:08:59.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:08:59.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:08:56cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621735,"remote_snapshot_timestamp":1771621735,"replay_state":"idle"}2026-02-20 21:08:58' 2026-02-20T21:08:59.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:08:59.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:08:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:56' 2026-02-20T21:08:59.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:08:59.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:08:59.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:08:59.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:08:59.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:08:59.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:59.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:08:59.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:08:59.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:08:59.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:08:59.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:08:59.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:08:59.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:08:59.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:08:59.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:59.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:09:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:09:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:09:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:09:00.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:09:00.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:09:00.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:09:00.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621735,"remote_snapshot_timestamp":1771621735,"replay_state":"idle"} 2026-02-20T21:09:00.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:09:00.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:58 2026-02-20T21:09:00.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:09:00.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:09:00.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:09:00.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:09:00.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:08:56' 2026-02-20T21:09:00.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:09:00.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:09:00.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:09:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621735,"remote_snapshot_timestamp":1771621735,"replay_state":"idle"} 2026-02-20T21:09:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:09:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:09:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:09:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:09:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:09:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:09:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:09:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:09:00.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:58' 2026-02-20T21:09:00.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:08:58' 2026-02-20T21:09:00.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:00.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:00.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:00.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:09:00.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:08:58' 2026-02-20T21:09:00.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:00.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:00.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:09:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621735,"remote_snapshot_timestamp":1771621735,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:08:58cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:08:56' 2026-02-20T21:09:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:09:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:09:00.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:08:58' 2026-02-20T21:09:00.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:09:00.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:09:00.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:09:00.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:09:00.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:09:00.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:00.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:00.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:09:00.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test 2026-02-20T21:09:00.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:00.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:00.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:00.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-20T21:09:01.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:09:01.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:09:01.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:01.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:01.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:01.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:09:01.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:01.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:01.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:01.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:09:01.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:09:01.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:01.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:09:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:09:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:09:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:09:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:09:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:09:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:09:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:09:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:09:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:09:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:09:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:09:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:09:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:09:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:09:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:09:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:09:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:09:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:09:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:09:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:09:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:09:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:09:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:09:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:09:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:09:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:09:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:09:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:09:02.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:09:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:09:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:09:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:09:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:09:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:09:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:09:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:09:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:09:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:09:02.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:09:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:02.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:09:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:09:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:09:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:09:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:09:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:09:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:09:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:09:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:09:04.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:09:04.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:04.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:04.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:04.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:09:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:09:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:09:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:09:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:09:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:09:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:09:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:09:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:09:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:09:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:09:04.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:09:04.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:04.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:09:04.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:09:04.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:09:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:09:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:09:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:09:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:09:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:09:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:09:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:09:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:09:04.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:09:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:09:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:09:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:09:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:09:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:09:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:09:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:09:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:09:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:09:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:09:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:09:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:09:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:09:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:09:04.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:04.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:04.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:04.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:09:04.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:09:04.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:09:04.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:09:04.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:09:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:09:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:09:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:09:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:09:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:09:05.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:09:05.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:09:05.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:09:05.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:09:05.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:09:05.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:03 2026-02-20T21:09:05.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:09:05.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:09:05.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:09:05.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:09:05.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:01' 2026-02-20T21:09:05.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:09:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:09:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:09:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:09:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:09:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:09:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:09:05.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:09:05.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:09:05.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:09:05.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:09:05.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:09:05.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:03' 2026-02-20T21:09:05.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:09:03' 2026-02-20T21:09:05.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:09:03' 2026-02-20T21:09:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:09:05.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:09:03cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:09:01' 2026-02-20T21:09:05.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:09:05.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:09:05.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:03' 2026-02-20T21:09:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:09:05.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:09:05.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:09:05.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:09:05.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:09:05.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:05.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:05.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:09:05.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:05.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:05.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:05.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:05.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:05.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:05.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:05.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:09:06.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:09:06.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:06.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:06.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:06.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:06.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:06.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:06.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:06.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:09:06.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:09:06.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:09:06.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:09:06.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:09:06.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:09:06.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:01 2026-02-20T21:09:06.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:09:06.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:09:06.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:09:06.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:09:06.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:03' 2026-02-20T21:09:06.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:09:06.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:09:06.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:09:06.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:09:06.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:09:06.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:09:06.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:09:06.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:09:06.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:09:06.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:09:06.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:09:06.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:09:06.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:01' 2026-02-20T21:09:06.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:09:01' 2026-02-20T21:09:06.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:06.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:06.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:06.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:06.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:09:01' 2026-02-20T21:09:06.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:06.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:06.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:09:06.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:09:01cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:09:03' 2026-02-20T21:09:06.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:09:06.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:09:06.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:01' 2026-02-20T21:09:06.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:09:06.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:09:06.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:09:06.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:09:06.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:09:06.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test 2026-02-20T21:09:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:09:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-20T21:09:07.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:09:07.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-20T21:09:07.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:07.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:07.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:07.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-20T21:09:07.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:07.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:07.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:07.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:09:07.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:09:07.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:07.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:09:08.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:09:08.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:08.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:09:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:09:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:09:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:09:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:09:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:09:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:09:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:09:08.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:09:08.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:08.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:09:08.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:09:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:09:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:09:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:09:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:09:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:09:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:09:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:09:08.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:09:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:09:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:09:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:09:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:09:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:09:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:09:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:09:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:09:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:09:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:09:08.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:09:08.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:09:08.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:09:08.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:08.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:08.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:09:08.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:09:08.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:09:08.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:09:08.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:09:08.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:09:08.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:08.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:08.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:08.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:09:08.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:09:08.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:09:08.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:09:08.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:08.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:09:08.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:09:08.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:08.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:09:08.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:09:08.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:09:08.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:09:08.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:09:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:09:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:09:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:09:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:09:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:09:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:09:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:09:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:09:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:09:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:09:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:09:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:09:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:09:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:09:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:09:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:09:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:09:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:09:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:09:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:09:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:09:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:09:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:09:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:09:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:09:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:09:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:09:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:09:11.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:09:11.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:11.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:11.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:11.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:09:11.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:09:11.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:09:11.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:09:11.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:09:11.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:09:11.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:11.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:09:11.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:09:11.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:09:11.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:11.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:09:11.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:09:11.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:09:11.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:11.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:09:11.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:09:11.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:11.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:09:11.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:09:11.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:09:11.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:09:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:09:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:09:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:09:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:09:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:09:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:09:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:09:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:09:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:09:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:09:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:09:11.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:09:11.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:09:11.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:09:11.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:11.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:11.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:09:11.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:09:11.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:09:11.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:09:11.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:09:11.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:09:11.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:11.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:11.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:11.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:09:11.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:09:11.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:09:11.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:09:11.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:11.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:09:11.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:09:11.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:11.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:09:11.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:09:11.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:09:11.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:09:11.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:09:11.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:09:11.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:09:11.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:09:11.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:09:11.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:11.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:09:11.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:09:11.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:09:11.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:09:11.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:09:11.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:09:11.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:09:11.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:09:11.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:09:11.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:09:11.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:11.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:09:11.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:09:11.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:09:11.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:09:11.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:09:11.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:09:11.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:09:11.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:11.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:11.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:11.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:09:11.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:09:11.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:09:11.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:09:11.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:09:11.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:09:11.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:11.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:09:15.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:09:15.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:15.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:15.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:15.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:09:15.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:09:15.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:09:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:09:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:09:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:09:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:09:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:09:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:09:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:15.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:09:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:09:15.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:09:15.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:15.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:09:15.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:09:15.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:15.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:09:15.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:09:15.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:09:15.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:09:15.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:09:15.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:09:15.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:09:15.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:09:15.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:09:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:09:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:09:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:09:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:09:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:09:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:09:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:09:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:09:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:09:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:15.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:09:15.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:09:15.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:09:15.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:09:15.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:09:15.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:09:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:15.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:15.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:09:15.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:09:15.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:09:15.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:09:15.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:15.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:09:15.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:09:15.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:15.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:09:15.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:09:15.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:09:15.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:09:15.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:09:15.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:09:15.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:09:15.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:09:15.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:09:15.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:15.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:09:15.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:09:15.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:09:15.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:09:15.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:09:15.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:09:15.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:09:15.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:09:15.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:09:15.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:09:15.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:15.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:09:15.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:09:15.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:09:15.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:09:15.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:09:15.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:09:15.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:09:15.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:15.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:15.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:15.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:09:15.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:09:15.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:09:15.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:09:15.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:09:15.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:09:15.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:09:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:09:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:09:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:09:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:09:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:09:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:09:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:09:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-20T21:09:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-20T21:09:18.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 51 2026-02-20T21:09:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:09:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:09:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:09:18.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:09:18.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:18.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:09:18.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:09:18.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:09:18.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:09:18.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:09:18.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:09:18.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:09:18.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:09:18.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:09:18.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:18.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:18.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:18.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:09:18.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:09:18.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:09:18.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:09:18.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=51 2026-02-20T21:09:18.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:09:18.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:09:18.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:09:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:09:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:09:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:09:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:09:19.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:09:19.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=51 2026-02-20T21:09:19.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 51 = 51 2026-02-20T21:09:19.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:19.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:09:19.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:19.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:19.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:19.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:09:19.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:19.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:19.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:19.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:09:20.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:09:20.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:20.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:20.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:20.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:09:20.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:20.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:20.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:20.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:20.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:09:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:09:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:09:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:09:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:09:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:09:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:18 2026-02-20T21:09:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:09:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:09:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:09:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621746,"replay_state":"idle"} 2026-02-20T21:09:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:11 2026-02-20T21:09:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:09:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 51 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.3a3ce862-d191-4dc8-b1e7-11aa8e675a9b (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:09:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:09:20.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:09:20.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:09:20.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:09:20.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:09:20.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:09:20.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:09:20.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:09:20.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:09:20.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:09:20.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:09:20.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:09:20.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:18' 2026-02-20T21:09:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:09:18' 2026-02-20T21:09:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:09:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:09:18' 2026-02-20T21:09:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:09:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary44044409mirror.0trial1992026-02-20 21:09:18cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621746,"replay_state":"idle"}2026-02-20 21:09:11' 2026-02-20T21:09:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:09:20.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:09:20.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:18' 2026-02-20T21:09:20.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:09:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:09:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:09:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:09:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:09:20.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:20.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:20.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-20T21:09:20.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:20.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:20.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:20.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:09:20.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:20.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:20.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:20.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:09:21.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-20T21:09:21.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:21.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:21.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:21.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:09:21.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:21.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:21.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:21.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:21.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:09:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:09:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:09:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:09:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621746,"replay_state":"idle"} 2026-02-20T21:09:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:09:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:11 2026-02-20T21:09:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:09:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:09:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:09:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:09:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:18' 2026-02-20T21:09:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:09:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:09:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:09:21.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621746,"replay_state":"idle"} 2026-02-20T21:09:21.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:09:21.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:09:21.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:09:21.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:09:21.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:09:21.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:09:21.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:09:21.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:09:21.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:11' 2026-02-20T21:09:21.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-20 21:09:11' 2026-02-20T21:09:21.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:21.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:21.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:21.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:09:21.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:09:11' 2026-02-20T21:09:21.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:21.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:21.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:09:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621746,"replay_state":"idle"}45294534mirror.1trial1992026-02-20 21:09:11cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:09:18' 2026-02-20T21:09:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:09:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:09:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:11' 2026-02-20T21:09:21.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:09:21.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:09:21.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:09:21.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:09:21.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:09:21.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:21.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:21.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster1 mirror test 2026-02-20T21:09:21.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:21.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:21.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:21.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-20T21:09:21.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:09:21.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-20T21:09:21.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:21.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:21.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:21.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-20T21:09:21.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:21.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:21.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:21.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:09:21.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:09:21.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:21.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:09:22.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:09:22.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:22.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:22.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:22.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:09:22.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:09:22.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:09:22.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:09:22.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:09:22.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:09:22.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:22.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:09:22.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:09:22.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:09:22.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:22.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:09:22.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:09:22.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:09:22.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:22.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:09:22.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:09:22.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:22.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:09:22.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:09:22.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:09:22.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:09:22.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:09:22.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:09:22.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:09:22.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:09:22.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:09:22.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:22.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:09:22.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:22.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:09:22.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:09:22.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:09:22.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:09:22.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:09:22.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:09:22.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:09:22.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:09:22.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:22.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:22.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:09:22.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:09:22.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:09:22.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:09:22.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:09:22.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:09:22.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:22.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:22.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:22.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:09:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:09:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:09:22.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:09:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:09:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:09:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:09:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:09:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:09:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:09:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:09:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:09:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:09:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:09:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:09:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:09:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:09:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:09:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:09:22.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:09:22.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:09:22.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:09:22.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:09:22.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:09:22.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:09:22.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:22.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:09:22.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:09:22.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:09:22.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:09:22.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:09:22.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:09:22.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:09:22.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:22.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:22.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:22.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:09:23.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:09:23.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:09:23.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:09:23.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:09:23.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:09:23.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:09:23.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:09:23.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:23.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:09:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:09:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:09:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:09:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:09:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:09:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:09:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:09:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:09:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:09:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:09:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:25.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:09:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:09:25.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:09:25.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:25.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:09:25.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:09:25.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:25.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:09:25.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:09:25.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:09:25.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:09:25.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:09:25.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:09:25.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:09:25.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:09:25.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:09:25.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:25.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:09:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:09:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:09:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:09:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:09:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:09:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:09:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:09:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:09:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:25.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:25.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:09:25.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:09:25.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:09:25.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:09:25.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:09:25.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:09:25.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:25.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:25.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:25.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:09:25.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:09:25.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:09:25.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:09:25.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:25.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:09:25.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:09:25.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:25.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:09:25.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:09:25.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:09:25.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:09:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:09:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:09:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:09:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:09:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:09:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:09:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:09:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:09:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:09:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:09:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:09:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:09:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:09:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:09:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:09:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:09:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:09:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:09:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:09:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:09:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:09:25.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:09:25.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:25.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:25.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:25.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:09:25.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:09:25.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:09:25.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:09:25.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:09:25.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:09:25.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:25.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:09:25.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:25.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:25.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:25.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:25.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:25.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:25.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:25.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:09:26.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:09:26.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:26.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:26.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:26.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:26.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:26.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:26.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:26.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:26.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:09:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:09:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:09:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:09:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:09:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:09:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:23 2026-02-20T21:09:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:09:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:09:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:09:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:09:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:23' 2026-02-20T21:09:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:09:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:09:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:09:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:09:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:09:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:09:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:09:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:09:26.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:09:26.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:09:26.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:09:26.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:09:26.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:23' 2026-02-20T21:09:26.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:09:23' 2026-02-20T21:09:26.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:26.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:26.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:26.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:26.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:09:23' 2026-02-20T21:09:26.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:26.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:26.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:09:26.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:09:23cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image demoted2026-02-20 21:09:23' 2026-02-20T21:09:26.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:09:26.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:09:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:23' 2026-02-20T21:09:26.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:09:26.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:09:26.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:09:26.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:09:26.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:09:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:09:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:09:27.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:09:27.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:27.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:27.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:27.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:27.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:27.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:27.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:27.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:27.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:09:27.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:09:27.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:09:27.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:09:27.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:09:27.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:09:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:26 2026-02-20T21:09:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:09:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:09:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:09:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:09:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:23' 2026-02-20T21:09:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:09:27.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:09:27.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:09:27.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:09:27.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:09:27.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:09:27.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:09:27.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:09:27.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:09:27.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:09:27.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:09:27.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:09:27.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:26' 2026-02-20T21:09:27.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:09:26' 2026-02-20T21:09:27.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:27.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:27.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:27.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:27.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:09:26' 2026-02-20T21:09:27.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:27.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:27.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:09:27.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:09:26cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:09:23' 2026-02-20T21:09:27.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:09:27.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:09:27.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:26' 2026-02-20T21:09:27.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:09:27.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:09:27.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:09:27.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:09:27.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:09:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster2 mirror test 2026-02-20T21:09:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:09:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-20T21:09:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:09:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:09:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:09:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:09:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:09:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:29.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:09:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:09:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:09:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:09:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:09:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:09:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:09:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:09:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:30.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:09:30.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:09:30.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:09:30.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:30.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:09:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:09:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:09:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:09:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:09:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:09:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:09:30.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:09:30.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:09:30.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:09:30.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:09:30.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:09:30.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:30.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:09:30.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:09:30.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:09:30.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:30.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:09:30.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:09:30.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:09:30.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:09:30.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:09:30.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:09:30.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:09:30.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:09:30.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:09:30.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:30.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:30.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:09:30.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:09:30.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:09:30.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:09:30.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:09:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:09:30.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:09:30.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:09:30.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:09:30.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:09:30.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:09:30.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:09:30.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:09:30.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:09:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:09:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:09:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:09:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:09:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:09:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:09:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:30.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:30.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:09:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 54 2026-02-20T21:09:34.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:09:34.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:09:34.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:34.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:09:34.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:09:34.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:34.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:09:34.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:09:34.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:09:34.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:09:34.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:09:34.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:09:34.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:09:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:09:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:09:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:34.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:09:34.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:09:34.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:09:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:09:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=54 2026-02-20T21:09:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:09:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:09:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:09:34.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:09:34.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:34.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:34.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:34.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:09:34.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:09:34.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:09:34.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:09:34.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=54 2026-02-20T21:09:34.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 54 = 54 2026-02-20T21:09:34.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:34.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-20T21:09:34.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:34.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:34.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:34.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:09:34.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:34.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:34.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:34.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:09:35.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-20T21:09:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:09:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:09:35.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:09:35.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:09:35.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:09:35.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:09:35.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:09:35.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:31 2026-02-20T21:09:35.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:09:35.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:09:35.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:09:35.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621768,"replay_state":"idle"} 2026-02-20T21:09:35.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:28 2026-02-20T21:09:35.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:09:35.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 54 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.ff2b42af-d43d-4310-b9fe-3fb7d6d483f8 (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:09:35.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:09:35.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:09:35.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:09:35.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:09:35.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:09:35.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:09:35.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:09:35.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:09:35.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:09:35.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:09:35.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:09:35.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:09:35.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:31' 2026-02-20T21:09:35.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-20 21:09:31' 2026-02-20T21:09:35.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:35.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:35.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:35.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:09:35.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:09:31' 2026-02-20T21:09:35.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:35.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:35.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:09:35.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:09:31cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621768,"replay_state":"idle"}2026-02-20 21:09:28' 2026-02-20T21:09:35.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:09:35.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:09:35.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:31' 2026-02-20T21:09:35.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:09:35.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:09:35.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:09:35.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:09:35.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:09:35.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:35.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:35.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:09:35.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:35.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:35.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:35.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:09:35.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:35.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:35.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:35.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:09:36.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:09:36.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:36.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:36.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:36.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:09:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:09:36.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:09:36.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:09:36.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:09:36.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621768,"replay_state":"idle"} 2026-02-20T21:09:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:09:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:28 2026-02-20T21:09:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:09:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:09:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:09:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:09:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:31' 2026-02-20T21:09:36.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:09:36.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:09:36.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:09:36.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621768,"replay_state":"idle"} 2026-02-20T21:09:36.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:09:36.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:09:36.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:09:36.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:09:36.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:09:36.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:09:36.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:09:36.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:09:36.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:28' 2026-02-20T21:09:36.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:09:28' 2026-02-20T21:09:36.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:36.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:36.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:36.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:09:36.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:09:28' 2026-02-20T21:09:36.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:36.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:36.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:09:36.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621768,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:09:28cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:09:31' 2026-02-20T21:09:36.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:09:36.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:09:36.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:28' 2026-02-20T21:09:36.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:09:36.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:09:36.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:09:36.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:09:36.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:09:36.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:36.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:09:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test 2026-02-20T21:09:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:36.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-20T21:09:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:09:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:09:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:09:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:09:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:09:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:09:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:09:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:09:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:09:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:09:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:09:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:09:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:09:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:09:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:09:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:09:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:09:38.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:09:38.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:09:38.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:38.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:09:38.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:09:38.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:38.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:09:38.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:09:38.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:09:38.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:09:38.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:09:38.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:09:38.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:09:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:09:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:09:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:09:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:09:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:09:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:09:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:09:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:09:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:09:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:09:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:09:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:09:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:09:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:09:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:09:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:09:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:09:38.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:38.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:38.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:38.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:09:38.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:09:38.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:09:38.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:09:38.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:09:38.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:09:38.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:09:38.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:09:38.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:38.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:09:40.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:09:40.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:40.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:40.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:40.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:09:40.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:09:40.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:09:40.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:09:40.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:09:40.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:09:40.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:40.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:09:40.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:09:40.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:09:40.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:40.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:09:40.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:09:40.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:09:40.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:40.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:09:40.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:09:40.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:40.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:09:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:09:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:09:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:09:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:09:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:09:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:09:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:09:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:09:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:09:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:40.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:09:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:09:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:09:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:09:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:09:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:09:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:09:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:09:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:09:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:09:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:09:40.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:09:40.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:09:40.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:09:40.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:40.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:09:40.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:09:40.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:09:40.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:09:40.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:09:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:09:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:09:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:09:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:09:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:41.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:41.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:09:41.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:09:41.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:09:41.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:09:41.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:09:41.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:09:41.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:38 2026-02-20T21:09:41.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:09:41.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:09:41.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:09:41.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:09:41.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:36' 2026-02-20T21:09:41.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:09:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:09:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:09:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image demoted 2026-02-20T21:09:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:09:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:09:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:09:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:09:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:09:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:09:41.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:09:41.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:09:41.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:38' 2026-02-20T21:09:41.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:09:38' 2026-02-20T21:09:41.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:41.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:41.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:41.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:41.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:09:38' 2026-02-20T21:09:41.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:41.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:41.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:09:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image demoted44044409mirror.0trial1992026-02-20 21:09:38cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:09:36' 2026-02-20T21:09:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:09:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:09:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:38' 2026-02-20T21:09:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:09:41.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:09:41.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:09:41.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:09:41.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:09:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:09:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:09:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:09:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:42.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:42.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:42.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:42.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:42.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:42.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:42.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:09:42.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:09:42.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:09:42.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:09:42.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:09:42.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:09:42.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:41 2026-02-20T21:09:42.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:09:42.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:09:42.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:09:42.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:09:42.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:38' 2026-02-20T21:09:42.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:09:42.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:09:42.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:09:42.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:09:42.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:09:42.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:09:42.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:09:42.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:09:42.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:09:42.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:09:42.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:09:42.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:09:42.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:41' 2026-02-20T21:09:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:09:41' 2026-02-20T21:09:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:09:41' 2026-02-20T21:09:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:09:42.569 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:09:41cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image demoted2026-02-20 21:09:38' 2026-02-20T21:09:42.569 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:09:42.569 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:09:42.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:41' 2026-02-20T21:09:42.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:09:42.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:09:42.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:09:42.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:09:42.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:09:42.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:42.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:42.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test 2026-02-20T21:09:42.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:42.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:42.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:42.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:09:42.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-20T21:09:43.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:09:43.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-20T21:09:43.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:43.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:43.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:43.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-20T21:09:43.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:43.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:43.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:43.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:09:43.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:09:43.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:43.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:09:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:09:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:09:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:09:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:09:44.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:09:44.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:09:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:09:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:09:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:09:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:09:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:09:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:09:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:09:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:09:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:09:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:09:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:09:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:09:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:09:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:09:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:09:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:09:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:09:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:09:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:09:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:09:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:09:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:09:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:09:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:09:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:09:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:09:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:09:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:09:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:09:44.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:09:44.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:09:44.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:09:44.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:09:44.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:44.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:44.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:44.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:09:44.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:09:44.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:09:44.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:09:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:09:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:09:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:09:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:09:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:09:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:09:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:09:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:09:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:09:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:09:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:09:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:09:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:09:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:09:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:09:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:09:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:09:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:09:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:09:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:09:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:09:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:09:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:09:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:09:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:09:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:09:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:09:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:09:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:44.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:09:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:09:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:09:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:09:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:09:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:09:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:09:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:09:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:09:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:09:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:09:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:09:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:09:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:09:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:09:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:09:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:09:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:09:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:09:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:09:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:09:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:09:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:09:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:09:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:09:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:09:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:09:46.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:09:46.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:09:46.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:09:46.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:09:46.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:46.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:09:46.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:46.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:09:46.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:09:46.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:09:46.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:09:46.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:09:46.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:09:46.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:09:46.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:09:46.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:46.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:46.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:09:46.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:09:46.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:09:46.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:09:46.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:09:46.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:09:47.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:47.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:47.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:47.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:09:47.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:09:47.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:09:47.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:09:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:09:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:09:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:09:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:09:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:09:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:09:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:09:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:09:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:09:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:09:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:09:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:09:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:09:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:09:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:09:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:09:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:09:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:09:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:09:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:09:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:09:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:09:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:09:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:09:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:09:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:09:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:09:47.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:09:47.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:47.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:47.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:47.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:09:47.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:09:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:09:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:09:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:09:47.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:09:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:09:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:09:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:09:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:09:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:09:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:09:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:09:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:09:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-20T21:09:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-20T21:09:50.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 57 2026-02-20T21:09:50.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:09:50.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:09:50.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:50.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:09:50.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:09:50.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:50.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:09:50.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:09:50.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:09:50.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:09:50.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:09:50.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:09:50.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:09:50.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:09:50.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:09:50.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:50.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:50.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:50.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:09:50.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:09:50.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:09:50.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:09:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=57 2026-02-20T21:09:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:09:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:09:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:09:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:09:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:51.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:09:51.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:09:51.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:09:51.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:09:51.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=57 2026-02-20T21:09:51.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 57 = 57 2026-02-20T21:09:51.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:51.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:09:51.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:51.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:51.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:51.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:09:51.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:51.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:51.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:51.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:09:52.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:09:52.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:52.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:52.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:52.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:09:52.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:52.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:52.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:52.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:09:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:09:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:09:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:09:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:09:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:09:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:48 2026-02-20T21:09:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:09:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:09:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:09:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621787,"remote_snapshot_timestamp":1771621787,"replay_state":"idle"} 2026-02-20T21:09:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:51 2026-02-20T21:09:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:09:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 57 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.6c99ce4b-61cb-404f-a5f8-e0ae9b03abf7 (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:09:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:09:52.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:09:52.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:09:52.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:09:52.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:09:52.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:09:52.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:09:52.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:09:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:09:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:09:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:09:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:09:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:48' 2026-02-20T21:09:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:09:48' 2026-02-20T21:09:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:09:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:09:48' 2026-02-20T21:09:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:09:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary44044409mirror.0trial1992026-02-20 21:09:48cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621787,"remote_snapshot_timestamp":1771621787,"replay_state":"idle"}2026-02-20 21:09:51' 2026-02-20T21:09:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:09:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:09:52.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:48' 2026-02-20T21:09:52.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:09:52.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:09:52.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:09:52.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:09:52.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:09:52.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:52.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:52.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-20T21:09:52.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:52.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:52.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:09:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:09:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-20T21:09:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:09:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:53.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:53.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:09:53.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:09:53.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:09:53.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:09:53.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621787,"remote_snapshot_timestamp":1771621787,"replay_state":"idle"} 2026-02-20T21:09:53.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:09:53.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:51 2026-02-20T21:09:53.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:09:53.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:09:53.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:09:53.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:09:53.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:53' 2026-02-20T21:09:53.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:09:53.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:09:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:09:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621787,"remote_snapshot_timestamp":1771621787,"replay_state":"idle"} 2026-02-20T21:09:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:09:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:09:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:09:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:09:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:09:53.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:09:53.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:09:53.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:09:53.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:51' 2026-02-20T21:09:53.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-20 21:09:51' 2026-02-20T21:09:53.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:53.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:53.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:53.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:09:53.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:09:51' 2026-02-20T21:09:53.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:53.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:53.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:09:53.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621787,"remote_snapshot_timestamp":1771621787,"replay_state":"idle"}45294534mirror.1trial1992026-02-20 21:09:51cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:09:53' 2026-02-20T21:09:53.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:09:53.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:09:53.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:51' 2026-02-20T21:09:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:09:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:09:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:09:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:09:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:09:53.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:53.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:53.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster1 mirror test 2026-02-20T21:09:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-20T21:09:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:09:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-20T21:09:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-20T21:09:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:09:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:09:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:09:54.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:09:54.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:54.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:54.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:54.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:09:54.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:09:54.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:09:54.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:09:54.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:09:54.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:09:54.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:54.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:09:54.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:09:54.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:09:54.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:54.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:09:54.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:09:54.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:09:54.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:54.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:09:54.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:09:54.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:54.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:09:54.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:09:54.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:09:54.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:09:54.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:09:54.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:09:54.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:09:54.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:09:54.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:09:54.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:54.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:09:54.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:54.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:09:54.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:09:54.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:09:54.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:09:54.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:09:54.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:09:54.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:09:54.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:09:54.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:54.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:54.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:09:54.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:09:54.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:09:54.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:09:54.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:09:54.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:09:54.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:54.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:54.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:54.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:09:55.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:09:55.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:09:55.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:09:55.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:55.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:09:55.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:09:55.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:55.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:09:55.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:09:55.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:09:55.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:09:55.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:09:55.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:09:55.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:09:55.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:09:55.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:09:55.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:55.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:09:55.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:09:55.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:09:55.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:09:55.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:09:55.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:09:55.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:09:55.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:09:55.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:09:55.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:09:55.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:55.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:09:55.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:09:55.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:09:55.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:09:55.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:09:55.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:09:55.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:09:55.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:55.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:55.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:55.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:09:55.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:09:55.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:09:55.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:09:55.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:09:55.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:09:55.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:09:55.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:09:55.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:55.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:09:57.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:09:57.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:57.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:57.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:57.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:09:57.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:09:57.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:09:57.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:09:57.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:09:57.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:09:57.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:57.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:09:57.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:09:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:09:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:09:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:09:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:09:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:09:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:09:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:09:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:09:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:09:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:09:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:09:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:09:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:09:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:09:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:09:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:09:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:09:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:09:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:09:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:09:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:09:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:09:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:09:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:09:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:09:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:09:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:09:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:09:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:09:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:09:57.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:09:57.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:57.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:57.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:09:57.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:09:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:09:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:09:57.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:09:57.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:57.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:09:57.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:09:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:09:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:09:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:09:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:09:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:09:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:09:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:09:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:09:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:09:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:09:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:09:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:09:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:09:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:09:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:09:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:09:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:09:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:09:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:09:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:09:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:09:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:09:57.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:09:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:09:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:09:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:09:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:09:57.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:09:57.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:57.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:57.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:09:57.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:09:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:09:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:09:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:09:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:09:57.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:09:57.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:57.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:09:57.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:57.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:57.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:57.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:57.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:57.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:57.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:57.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:09:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:09:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:58.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:58.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:58.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:09:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:09:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:09:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:09:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:09:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:09:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:58 2026-02-20T21:09:58.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:09:58.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:09:58.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:09:58.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:09:58.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:56' 2026-02-20T21:09:58.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:09:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:09:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:09:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:09:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:09:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:09:58.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:09:58.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:09:58.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:09:58.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:09:58.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:09:58.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:09:58.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:58' 2026-02-20T21:09:58.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:09:58' 2026-02-20T21:09:58.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:09:58.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:58.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:58.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:58.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:09:58' 2026-02-20T21:09:58.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:58.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:58.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:09:58.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:09:58cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:09:56' 2026-02-20T21:09:58.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:09:58.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:09:58.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:58' 2026-02-20T21:09:58.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:09:58.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:09:58.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:09:58.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:09:58.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:09:58.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:58.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:58.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:09:58.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:58.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:58.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:58.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:58.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:58.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:58.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:58.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:09:59.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:09:59.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:59.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:59.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:59.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:59.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:09:59.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:09:59.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:59.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:59.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:09:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:09:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:09:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:09:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:09:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:09:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:56 2026-02-20T21:09:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:09:59.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:09:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:09:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:09:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:09:58' 2026-02-20T21:09:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:09:59.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:09:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:09:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:09:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:09:59.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:09:59.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:09:59.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:09:59.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:09:59.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:09:59.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:09:59.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:09:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:56' 2026-02-20T21:09:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:09:56' 2026-02-20T21:09:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:09:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:09:56' 2026-02-20T21:09:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:09:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:09:59.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:09:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:09:56cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:09:58' 2026-02-20T21:09:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:09:59.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:09:59.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:09:56' 2026-02-20T21:09:59.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:09:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:09:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:09:59.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:09:59.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:09:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:09:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster2 mirror test 2026-02-20T21:09:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:09:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:09:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:09:59.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:09:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-20T21:10:01.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:10:01.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:10:01.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:01.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:01.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:01.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:10:01.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:01.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:01.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:01.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:10:01.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:10:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:01.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:10:02.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:10:02.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:02.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:02.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:02.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:10:02.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:10:02.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:10:02.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:10:02.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:10:02.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:02.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:10:02.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:10:02.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:10:02.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:02.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:10:02.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:10:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:10:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:10:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:10:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:10:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:10:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:10:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:10:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:10:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:10:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:10:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:10:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:10:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:10:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:10:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:10:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:10:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:10:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:10:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:10:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:10:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:10:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:10:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:10:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:02.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:02.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:02.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:02.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:10:02.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:10:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:10:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:10:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:10:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:10:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:10:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:10:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:10:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:10:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:10:04.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:10:04.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:10:04.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:04.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:10:04.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:10:04.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:10:04.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:04.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:10:04.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:10:04.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:04.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:10:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:10:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:10:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:10:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:10:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:10:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:10:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:10:04.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:10:04.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:04.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:10:04.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:10:04.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:10:04.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:04.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:04.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:10:04.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:10:04.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:10:04.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:04.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:10:04.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:10:04.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:10:04.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:10:04.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:04.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:04.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:10:04.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:10:04.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:10:04.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:10:04.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:04.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:04.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:04.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:04.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:10:04.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:10:04.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:10:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:10:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:10:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:10:04.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:10:04.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:10:04.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:04.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:10:04.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:10:04.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:04.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:10:04.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:10:04.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:04.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:10:04.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:10:04.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:10:04.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:04.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:04.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:04.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:10:08.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 60 2026-02-20T21:10:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:10:08.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:10:08.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:08.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:10:08.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:10:08.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:08.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:08.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:10:08.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:10:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:10:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:10:08.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:08.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:10:08.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:10:08.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:08.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:08.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:08.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:10:08.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:10:08.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:10:08.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:10:08.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=60 2026-02-20T21:10:08.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:10:08.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:08.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:10:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:10:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:10:08.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:10:08.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:10:08.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:10:08.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=60 2026-02-20T21:10:08.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 60 = 60 2026-02-20T21:10:08.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:08.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-20T21:10:08.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:08.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:08.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:08.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:10:08.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:08.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:08.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:08.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:10:09.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-20T21:10:09.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:09.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:09.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:09.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:10:09.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:09.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:09.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:09.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:09.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:10:09.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:10:09.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:10:09.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:10:09.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:10:09.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:10:09.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:06 2026-02-20T21:10:09.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:10:09.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:10:09.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:10:09.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621800,"replay_state":"idle"} 2026-02-20T21:10:09.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:03 2026-02-20T21:10:09.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:10:09.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 60 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.95c28af6-2143-41e9-8402-341636537a71 (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:10:09.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:10:09.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:10:09.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:10:09.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:10:09.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:10:09.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:10:09.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:10:09.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:10:09.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:10:09.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:10:09.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:10:09.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:10:09.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:06' 2026-02-20T21:10:09.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-20 21:10:06' 2026-02-20T21:10:09.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:09.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:09.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:09.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:10:09.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:10:06' 2026-02-20T21:10:09.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:09.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:09.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:10:09.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:10:06cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621800,"replay_state":"idle"}2026-02-20 21:10:03' 2026-02-20T21:10:09.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:10:09.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:10:09.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:06' 2026-02-20T21:10:09.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:10:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:10:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:10:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:10:09.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:10:09.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:09.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:09.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:10:09.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:09.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:09.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:09.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:10:09.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:09.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:09.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:09.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:10:10.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:10:10.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:10.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:10.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:10.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:10:10.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:10.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:10.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:10.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:10.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:10:10.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:10:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:10:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:10:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621800,"replay_state":"idle"} 2026-02-20T21:10:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:10:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:03 2026-02-20T21:10:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:10:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:10:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:10:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:10:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:06' 2026-02-20T21:10:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:10:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:10:10.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:10:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621800,"replay_state":"idle"} 2026-02-20T21:10:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:10:10.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:10:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:10:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:10:10.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:10:10.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:10:10.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:10:10.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:10:10.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:03' 2026-02-20T21:10:10.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:10:03' 2026-02-20T21:10:10.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:10.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:10.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:10.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:10:10.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:10:03' 2026-02-20T21:10:10.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:10.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:10.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:10:10.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621800,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:10:03cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:10:06' 2026-02-20T21:10:10.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:10:10.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:10:10.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:03' 2026-02-20T21:10:10.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:10:10.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:10:10.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:10:10.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:10:10.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:10:10.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:10.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:10.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:10:10.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test 2026-02-20T21:10:10.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:10.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:10.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:10.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-20T21:10:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:10:11.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:10:11.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:10:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:10:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:10:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:10:12.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:10:12.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:12.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:12.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:12.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:10:12.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:10:12.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:10:12.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:10:12.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:10:12.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:10:12.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:12.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:10:12.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:10:12.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:10:12.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:12.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:10:12.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:12.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:10:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:10:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:10:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:10:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:10:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:10:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:10:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:10:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:10:12.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:12.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:10:12.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:10:12.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:10:12.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:12.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:12.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:10:12.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:10:12.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:10:12.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:12.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:10:12.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:10:12.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:10:12.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:10:12.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:12.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:12.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:10:12.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:10:12.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:10:12.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:10:12.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:12.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:12.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:12.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:12.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:10:12.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:10:12.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:10:12.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:10:12.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:10:12.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:10:12.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:10:12.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:10:12.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:12.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:10:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:10:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:10:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:10:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:10:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:10:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:10:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:10:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:10:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:10:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:10:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:10:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:10:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:10:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:10:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:10:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:10:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:10:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:10:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:10:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:10:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:14.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:10:14.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:10:14.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:10:14.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:14.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:14.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:10:14.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:10:14.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:10:14.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:14.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:10:14.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:10:14.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:10:14.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:10:14.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:14.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:14.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:10:14.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:10:14.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:10:14.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:10:14.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:14.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:14.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:14.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:14.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:10:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:10:14.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:10:14.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:10:14.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:10:14.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:10:14.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:14.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:10:14.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:14.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:14.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:14.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:10:14.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:14.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:14.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:14.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:10:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:10:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:10:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:10:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:10:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:10:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:10:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:10:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:10:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:13 2026-02-20T21:10:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:10:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:10:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:10:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:10:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:11' 2026-02-20T21:10:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:10:15.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:10:15.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:10:15.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:10:15.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:10:15.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:10:15.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:10:15.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:10:15.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:10:15.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:10:15.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:10:15.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:10:15.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:13' 2026-02-20T21:10:15.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:10:13' 2026-02-20T21:10:15.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:15.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:15.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:10:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:10:13' 2026-02-20T21:10:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:10:15.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:10:13cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:10:11' 2026-02-20T21:10:15.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:10:15.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:10:15.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:13' 2026-02-20T21:10:15.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:10:15.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:10:15.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:10:15.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:10:15.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:10:15.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:15.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:15.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:10:15.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:15.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:15.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:15.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:10:15.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:15.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:15.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:15.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:10:16.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:10:16.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:16.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:16.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:16.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:10:16.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:16.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:16.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:16.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:16.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:10:16.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:10:16.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:10:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:10:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:10:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:10:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:11 2026-02-20T21:10:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:10:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:10:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:10:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:10:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:13' 2026-02-20T21:10:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:10:16.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:10:16.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:10:16.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:10:16.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:10:16.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:10:16.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:10:16.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:10:16.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:10:16.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:10:16.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:10:16.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:10:16.353 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:11' 2026-02-20T21:10:16.353 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:10:11' 2026-02-20T21:10:16.353 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:16.353 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:16.353 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:16.353 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:10:16.353 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:10:11' 2026-02-20T21:10:16.353 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:16.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:16.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:10:16.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:10:11cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:10:13' 2026-02-20T21:10:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:10:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:10:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:11' 2026-02-20T21:10:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:10:16.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:10:16.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:10:16.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:10:16.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:10:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test 2026-02-20T21:10:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:10:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-20T21:10:17.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:10:17.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-20T21:10:17.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:17.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:17.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:17.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-20T21:10:17.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:17.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:17.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:17.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:10:17.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:10:17.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:17.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:10:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:10:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:10:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:10:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:10:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:10:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:10:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:10:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:10:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:10:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:10:18.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:18.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:10:18.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:10:18.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:10:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:10:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:10:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:10:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:10:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:10:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:10:18.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:10:18.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:10:18.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:10:18.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:10:18.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:18.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:10:18.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:10:18.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:18.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:10:18.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:10:18.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:18.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:10:18.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:10:18.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:10:18.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:10:18.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:10:18.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:10:18.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:10:18.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:10:18.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:10:18.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:18.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:10:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:10:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:10:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:10:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:10:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:10:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:10:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:10:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:10:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:10:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:10:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:10:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:10:18.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:10:18.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:10:18.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:10:18.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:10:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:10:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:10:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:10:18.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:10:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:10:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:10:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:10:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:10:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:10:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:10:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:10:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:10:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:10:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:10:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:10:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:10:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:10:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:10:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:10:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:10:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:10:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:10:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:10:21.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:21.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:10:21.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:10:21.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:21.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:10:21.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:10:21.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:10:21.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:10:21.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:10:21.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:10:21.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:10:21.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:10:21.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:10:21.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:21.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:10:21.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:21.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:10:21.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:10:21.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:10:21.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:10:21.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:10:21.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:10:21.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:10:21.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:10:21.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:21.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:21.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:10:21.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:10:21.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:10:21.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:10:21.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:10:21.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:10:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:10:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:10:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:10:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:10:21.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:21.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:10:21.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:10:21.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:21.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:10:21.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:10:21.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:21.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:10:21.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:10:21.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:10:21.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:10:21.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:10:21.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:10:21.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:10:21.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:10:21.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:10:21.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:21.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:10:21.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:10:21.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:10:21.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:10:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:10:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:10:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:10:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:10:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:10:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:10:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:10:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:10:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:10:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:10:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:10:21.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:10:21.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:10:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:10:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:10:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:10:21.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:10:21.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:10:21.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:10:21.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:10:21.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:10:21.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:10:21.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:21.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:10:25.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:10:25.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:25.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:25.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:25.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:10:25.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:10:25.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:10:25.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:10:25.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:10:25.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:10:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:10:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:10:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:10:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:25.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:10:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:10:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:10:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:10:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:10:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:10:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:10:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:10:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:10:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:10:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:10:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:10:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:10:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:10:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:10:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:10:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:10:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:10:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:10:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:10:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:10:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:10:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:10:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:10:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:10:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:10:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:10:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:10:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:10:25.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:10:25.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:10:25.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:10:25.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:10:25.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:25.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:10:25.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:10:25.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:25.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:10:25.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:10:25.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:25.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:10:25.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:10:25.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:10:25.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:10:25.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:10:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:10:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:10:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:10:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:10:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:10:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:10:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:10:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:10:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:10:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:10:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:10:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:10:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:10:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:10:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:10:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:10:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:10:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:10:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:10:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:10:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:10:25.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:10:25.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:10:25.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:10:25.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:10:25.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:10:25.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:10:25.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:10:25.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:10:25.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:10:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:10:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:10:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:10:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:10:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:10:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:10:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:10:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:10:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-20T21:10:25.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:25.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:25.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:25.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-20T21:10:28.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 63 2026-02-20T21:10:28.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:10:28.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:10:28.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:28.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:10:28.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:10:28.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:28.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:10:28.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:10:28.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:28.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:10:28.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:28.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:10:28.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:10:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:10:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:10:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:10:28.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:10:28.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:10:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=63 2026-02-20T21:10:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:10:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:10:29.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:10:29.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:29.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:29.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:29.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:10:29.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:10:29.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:10:29.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:10:29.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=63 2026-02-20T21:10:29.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 63 = 63 2026-02-20T21:10:29.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:29.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:10:29.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:29.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:29.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:29.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:10:29.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:29.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:29.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:29.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:10:30.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:10:30.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:30.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:30.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:30.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:10:30.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:30.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:30.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:30.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:30.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:10:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:10:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:10:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:10:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:10:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:10:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:28 2026-02-20T21:10:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:10:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:10:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:10:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621816,"replay_state":"idle"} 2026-02-20T21:10:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:21 2026-02-20T21:10:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:10:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 63 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.3548f1c8-4893-4fb8-ad00-3eb7eb1a4ec9 (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:10:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:10:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:10:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:10:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:10:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:10:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:10:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:10:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:10:30.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:10:30.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:10:30.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:10:30.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:10:30.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:28' 2026-02-20T21:10:30.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:10:28' 2026-02-20T21:10:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:10:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:10:28' 2026-02-20T21:10:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:10:30.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary44044409mirror.0trial1992026-02-20 21:10:28cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621816,"replay_state":"idle"}2026-02-20 21:10:21' 2026-02-20T21:10:30.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:10:30.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:10:30.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:28' 2026-02-20T21:10:30.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:10:30.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:10:30.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:10:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:10:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:10:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-20T21:10:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:10:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:10:31.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-20T21:10:31.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:10:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:10:31.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:10:31.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:10:31.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:10:31.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621816,"replay_state":"idle"} 2026-02-20T21:10:31.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:10:31.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:21 2026-02-20T21:10:31.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:10:31.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:10:31.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:10:31.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:10:31.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:28' 2026-02-20T21:10:31.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:10:31.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:10:31.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:10:31.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621816,"replay_state":"idle"} 2026-02-20T21:10:31.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:10:31.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:10:31.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:10:31.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:10:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:10:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:10:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:10:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:10:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:21' 2026-02-20T21:10:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-20 21:10:21' 2026-02-20T21:10:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:31.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:31.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:31.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:10:31.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:10:21' 2026-02-20T21:10:31.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:31.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:31.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:10:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621816,"replay_state":"idle"}45294534mirror.1trial1992026-02-20 21:10:21cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:10:28' 2026-02-20T21:10:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:10:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:10:31.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:21' 2026-02-20T21:10:31.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:10:31.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:10:31.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:10:31.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:10:31.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:10:31.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:31.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:31.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster1 mirror test 2026-02-20T21:10:31.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:31.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:31.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:31.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-20T21:10:31.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:10:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-20T21:10:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-20T21:10:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:10:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:10:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:10:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:10:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:10:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:10:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:10:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:10:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:10:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:10:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:10:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:10:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:10:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:10:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:10:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:10:32.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:10:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:10:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:10:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:10:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:10:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:10:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:10:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:10:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:10:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:10:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:10:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:10:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:10:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:10:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:10:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:10:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:10:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:10:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:10:32.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:10:32.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:32.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:32.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:10:32.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:10:32.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:10:32.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:10:32.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:10:32.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:10:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:10:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:10:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:10:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:10:33.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:33.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:10:33.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:10:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:10:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:10:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:10:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:10:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:10:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:10:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:10:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:10:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:10:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:10:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:10:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:10:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:10:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:10:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:10:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:10:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:10:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:10:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:10:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:10:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:10:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:10:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:10:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:10:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:10:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:10:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:10:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:10:33.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:10:33.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:10:33.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:10:33.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:10:33.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:10:33.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:10:33.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:10:33.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:10:33.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:10:33.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:10:33.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:10:33.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:33.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:10:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:10:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:10:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:10:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:10:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:10:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:10:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:10:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:10:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:10:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:10:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:10:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:10:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:10:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:10:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:10:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:10:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:10:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:10:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:10:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:10:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:10:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:10:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:10:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:10:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:10:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:10:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:10:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:10:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:10:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:10:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:10:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:10:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:10:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:10:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:10:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:10:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:10:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:10:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:10:35.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:10:35.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:10:35.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:10:35.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:10:35.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:35.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:10:35.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:10:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:10:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:10:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:10:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:10:35.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:10:35.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:10:35.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:10:35.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:10:35.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:10:35.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:10:35.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:10:35.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:35.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:10:35.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:10:35.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:10:35.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:10:35.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:10:35.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:10:35.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:10:35.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:10:35.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:10:35.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:10:35.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:35.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:10:35.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:10:35.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:10:35.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:10:35.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:10:35.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:10:35.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:10:35.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:10:35.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:10:35.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:10:35.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:10:35.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:10:35.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:10:35.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:10:35.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:10:35.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:10:35.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:35.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:10:35.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:35.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:35.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:35.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:10:35.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:35.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:35.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:35.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:10:36.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:10:36.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:36.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:36.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:36.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:10:36.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:36.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:36.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:36.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:36.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:10:36.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:10:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:10:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:10:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:10:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:10:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:33 2026-02-20T21:10:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:10:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:10:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:10:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:10:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:33' 2026-02-20T21:10:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:10:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:10:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:10:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:10:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:10:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:10:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:10:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:10:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:10:36.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:10:36.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:10:36.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:10:36.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:33' 2026-02-20T21:10:36.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:10:33' 2026-02-20T21:10:36.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:36.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:36.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:36.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:10:36.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:10:33' 2026-02-20T21:10:36.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:36.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:36.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:10:36.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:10:33cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image demoted2026-02-20 21:10:33' 2026-02-20T21:10:36.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:10:36.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:10:36.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:33' 2026-02-20T21:10:36.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:10:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:10:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:10:36.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:10:36.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:10:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:10:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:10:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:10:37.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:10:37.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:37.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:37.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:37.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:10:37.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:37.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:37.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:37.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:37.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:10:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:10:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:10:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:10:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:10:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:10:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:36 2026-02-20T21:10:37.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:10:37.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:10:37.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:10:37.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:10:37.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:33' 2026-02-20T21:10:37.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:10:37.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:10:37.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:10:37.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:10:37.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:10:37.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:10:37.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:10:37.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:10:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:10:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:10:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:10:37.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:10:37.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:36' 2026-02-20T21:10:37.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:10:36' 2026-02-20T21:10:37.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:37.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:37.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:37.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:10:37.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:10:36' 2026-02-20T21:10:37.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:37.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:37.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:10:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:10:36cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:10:33' 2026-02-20T21:10:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:10:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:10:37.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:36' 2026-02-20T21:10:37.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:10:37.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:10:37.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:10:37.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:10:37.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:10:37.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster2 mirror test 2026-02-20T21:10:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:10:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-20T21:10:38.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:10:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:10:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:10:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:10:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:10:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:10:39.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:10:39.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:10:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:10:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:10:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:10:39.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:10:39.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:10:39.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:39.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:10:39.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:10:39.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:10:39.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:39.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:10:39.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:39.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:10:39.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:10:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:10:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:10:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:10:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:10:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:10:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:10:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:10:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:10:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:10:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:10:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:10:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:10:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:10:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:10:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:10:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:10:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:10:39.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:39.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:39.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:10:39.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:10:39.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:10:39.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:10:39.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:39.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:39.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:39.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:39.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:10:39.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:10:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:10:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:10:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:10:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:10:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:10:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:10:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:10:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:10:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:10:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:10:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:39.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:10:39.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:10:39.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:10:39.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:39.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:39.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:39.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:10:41.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 66 2026-02-20T21:10:41.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:10:41.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:10:41.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:10:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:10:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:10:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:10:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:10:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:10:41.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:41.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:10:41.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:10:41.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:41.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:41.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:41.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:10:41.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:10:41.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:10:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:10:41.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=66 2026-02-20T21:10:41.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:10:41.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:41.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:10:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:10:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:42.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:42.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:10:42.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:10:42.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:10:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:10:42.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=66 2026-02-20T21:10:42.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 66 = 66 2026-02-20T21:10:42.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:42.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-20T21:10:42.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:42.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:42.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:42.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:10:42.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:42.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:42.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:42.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:10:43.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-20T21:10:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:10:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:43.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:10:43.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:10:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:10:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:10:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:10:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:10:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:41 2026-02-20T21:10:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:10:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:10:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:10:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621837,"replay_state":"idle"} 2026-02-20T21:10:43.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:38 2026-02-20T21:10:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:10:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 65 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.a3b904e9-5ae5-4771-854b-e4be537b8705 (peer_uuids:[]) 2026-02-20T21:10:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 66 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.1a3a3423-6d20-48fb-ad9b-fa2ae88aafb0 (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:10:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:10:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:10:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:10:43.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:10:43.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:10:43.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:10:43.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:10:43.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:10:43.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:10:43.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:10:43.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:10:43.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:10:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:41' 2026-02-20T21:10:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-20 21:10:41' 2026-02-20T21:10:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:10:43.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:10:41' 2026-02-20T21:10:43.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:43.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:43.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:10:43.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:10:41cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621837,"replay_state":"idle"}2026-02-20 21:10:38' 2026-02-20T21:10:43.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:10:43.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:10:43.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:41' 2026-02-20T21:10:43.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:10:43.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:10:43.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:10:43.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:10:43.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:10:43.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:43.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:43.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:10:43.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:43.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:43.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:43.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:10:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:10:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:10:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:10:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:10:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:10:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:10:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:10:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621837,"replay_state":"idle"} 2026-02-20T21:10:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:10:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:38 2026-02-20T21:10:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:10:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:10:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:10:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:10:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:41' 2026-02-20T21:10:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:10:44.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:10:44.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:10:44.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621837,"replay_state":"idle"} 2026-02-20T21:10:44.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:10:44.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:10:44.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:10:44.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:10:44.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:10:44.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:10:44.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:10:44.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:10:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:38' 2026-02-20T21:10:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:10:38' 2026-02-20T21:10:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:10:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:10:38' 2026-02-20T21:10:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:10:44.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621837,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:10:38cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:10:41' 2026-02-20T21:10:44.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:10:44.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:10:44.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:38' 2026-02-20T21:10:44.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:10:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:10:44.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:10:44.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:10:44.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:10:44.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:10:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test 2026-02-20T21:10:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-20T21:10:46.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:10:46.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:10:46.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:10:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:10:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:10:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:10:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:10:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:10:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:10:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:10:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:10:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:10:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:10:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:10:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:10:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:10:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:10:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:10:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:10:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:10:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:10:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:10:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:10:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:10:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:10:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:10:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:10:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:10:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:10:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:10:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:10:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:10:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:10:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:10:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:10:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:10:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:10:47.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:10:47.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:10:47.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:10:47.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:47.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:47.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:10:47.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:10:47.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:10:47.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:10:47.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:10:47.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:10:47.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:47.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:10:47.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:47.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:47.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:47.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:10:47.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:47.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:47.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:10:48.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:10:48.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:48.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:10:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:10:48.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:10:48.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:10:48.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:10:48.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:10:48.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:10:48.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:46 2026-02-20T21:10:48.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:10:48.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:10:48.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:10:48.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:10:48.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:46' 2026-02-20T21:10:48.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:10:48.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:10:48.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:10:48.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image demoted 2026-02-20T21:10:48.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:10:48.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:10:48.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:10:48.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:10:48.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:10:48.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:10:48.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:10:48.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:10:48.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:46' 2026-02-20T21:10:48.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:10:46' 2026-02-20T21:10:48.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:48.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:48.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:48.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:10:48.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:10:46' 2026-02-20T21:10:48.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:48.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:48.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:10:48.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:10:48cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:10:46' 2026-02-20T21:10:48.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:10:48.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:10:48.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:48' 2026-02-20T21:10:48.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:10:48.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:10:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:10:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:10:48.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:10:48.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:48.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:48.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:10:48.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:48.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:48.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:48.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:10:48.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:48.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:48.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:48.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:10:49.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:10:49.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:49.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:49.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:49.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:10:49.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:49.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:49.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:10:49.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:10:49.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:10:49.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:10:49.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:10:49.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:10:49.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:46 2026-02-20T21:10:49.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:10:49.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:10:49.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:10:49.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:10:49.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:48' 2026-02-20T21:10:49.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:10:49.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:10:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:10:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:10:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:10:49.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:10:49.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:10:49.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:10:49.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:10:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:10:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:10:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:10:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:46' 2026-02-20T21:10:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:10:46' 2026-02-20T21:10:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:10:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:10:46' 2026-02-20T21:10:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:10:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:10:46cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:10:48' 2026-02-20T21:10:49.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:10:49.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:10:49.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:46' 2026-02-20T21:10:49.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:10:49.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:10:49.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:10:49.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:10:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:10:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test 2026-02-20T21:10:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:10:49.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-20T21:10:50.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:10:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-20T21:10:50.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-20T21:10:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:10:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:10:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:10:51.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:10:51.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:51.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:51.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:51.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:10:51.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:10:51.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:10:51.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:10:51.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:10:51.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:10:51.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:51.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:10:51.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:10:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:10:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:10:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:10:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:10:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:10:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:10:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:10:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:10:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:10:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:10:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:10:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:10:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:10:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:10:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:10:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:10:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:10:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:10:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:10:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:10:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:10:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:10:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:10:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:10:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:10:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:10:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:10:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:10:51.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:10:51.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:10:51.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:10:51.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:10:51.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:10:51.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:10:51.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:51.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:10:51.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:10:51.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:10:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:10:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:10:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:10:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:10:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:10:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:10:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:10:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:10:51.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:10:51.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:10:51.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:51.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:10:51.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:10:51.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:10:51.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:10:51.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:10:51.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:10:51.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:10:51.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:10:51.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:10:51.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:10:51.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:51.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:10:51.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:10:51.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:10:51.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:10:51.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:10:51.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:10:51.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:10:51.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:10:51.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:10:51.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:10:51.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:10:52.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:10:52.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:10:52.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:10:52.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:10:52.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:10:52.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:10:52.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:10:52.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:52.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:10:52.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:10:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:10:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:10:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:10:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:10:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-20T21:10:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-20T21:10:55.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 69 2026-02-20T21:10:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:10:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:10:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:10:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:10:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:10:55.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:10:55.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:55.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:10:55.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:55.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:10:55.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:55.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:10:55.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:10:55.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:55.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:55.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:55.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:10:55.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:10:55.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:10:55.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:10:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=69 2026-02-20T21:10:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:10:55.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:55.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:10:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:10:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:10:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:10:56.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:10:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:10:56.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=69 2026-02-20T21:10:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 69 = 69 2026-02-20T21:10:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:10:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:10:56.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:56.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:56.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:56.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:10:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:10:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:10:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:57.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:57.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:57.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:57.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:10:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:10:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:10:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:10:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:10:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:10:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:53 2026-02-20T21:10:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:10:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:10:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:10:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621849,"replay_state":"idle"} 2026-02-20T21:10:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:51 2026-02-20T21:10:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:10:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 69 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.87338b12-ab13-497d-b063-364baefdfa9a (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:10:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:10:57.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:10:57.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:10:57.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:10:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:10:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:10:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:10:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:10:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:10:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:10:57.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:10:57.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:10:57.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:53' 2026-02-20T21:10:57.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:10:53' 2026-02-20T21:10:57.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:57.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:57.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:57.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:10:57.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:10:53' 2026-02-20T21:10:57.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:57.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:57.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:10:57.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary44044409mirror.0trial1992026-02-20 21:10:53cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621849,"replay_state":"idle"}2026-02-20 21:10:51' 2026-02-20T21:10:57.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:10:57.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:10:57.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:53' 2026-02-20T21:10:57.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:10:57.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:10:57.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:10:57.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:10:57.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:10:57.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:57.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:57.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-20T21:10:57.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:57.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:57.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:57.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:10:57.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:57.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:57.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:57.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:10:58.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-20T21:10:58.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:58.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:58.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:58.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:10:58.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:10:58.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:10:58.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:58.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:58.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:10:58.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:10:58.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:10:58.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:10:58.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621849,"replay_state":"idle"} 2026-02-20T21:10:58.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:10:58.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:51 2026-02-20T21:10:58.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:10:58.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:10:58.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:10:58.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:10:58.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:58' 2026-02-20T21:10:58.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:10:58.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:10:58.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:10:58.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621849,"replay_state":"idle"} 2026-02-20T21:10:58.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:10:58.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:10:58.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:10:58.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:10:58.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:10:58.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:10:58.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:10:58.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:10:58.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:51' 2026-02-20T21:10:58.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-20 21:10:51' 2026-02-20T21:10:58.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:58.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:58.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:58.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:10:58.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:10:51' 2026-02-20T21:10:58.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:10:58.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:10:58.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:10:58.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621849,"replay_state":"idle"}45294534mirror.1trial1992026-02-20 21:10:51cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:10:58' 2026-02-20T21:10:58.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:10:58.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:10:58.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:10:51' 2026-02-20T21:10:58.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:10:58.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:10:58.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:10:58.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:10:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:10:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:10:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster1 mirror test 2026-02-20T21:10:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:10:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:58.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-20T21:10:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:10:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-20T21:10:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-20T21:10:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:10:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:10:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:10:59.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:10:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:10:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:10:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:10:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:10:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:10:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:10:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:10:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:10:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:10:59.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:59.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:10:59.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:10:59.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:10:59.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:59.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:10:59.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:10:59.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:10:59.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:59.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:10:59.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:10:59.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:59.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:10:59.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:10:59.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:10:59.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:10:59.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:10:59.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:10:59.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:10:59.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:10:59.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:10:59.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:59.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:10:59.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:59.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:10:59.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:10:59.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:10:59.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:10:59.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:10:59.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:10:59.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:10:59.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:10:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:10:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:10:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:10:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:10:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:10:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:10:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:10:59.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:10:59.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:10:59.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:10:59.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:10:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:10:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:10:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:10:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:10:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:10:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:10:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:10:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:10:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:10:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:10:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:10:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:10:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:10:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:10:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:10:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:10:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:10:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:10:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:10:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:10:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:10:59.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:10:59.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:10:59.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:10:59.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:10:59.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:10:59.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:10:59.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:10:59.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:10:59.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:10:59.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:10:59.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:10:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:10:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:10:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:10:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:11:00.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:11:00.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:11:00.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:11:00.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:11:00.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:11:00.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:11:00.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:11:00.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:00.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:11:02.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:11:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:11:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:11:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:11:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:11:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:11:02.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:11:02.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:02.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:11:02.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:11:02.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:11:02.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:02.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:11:02.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:11:02.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:11:02.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:02.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:11:02.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:11:02.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:02.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:11:02.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:11:02.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:11:02.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:11:02.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:11:02.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:11:02.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:11:02.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:11:02.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:11:02.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:02.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:11:02.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:02.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:11:02.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:11:02.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:11:02.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:11:02.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:11:02.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:11:02.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:11:02.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:11:02.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:02.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:02.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:11:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:11:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:11:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:11:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:11:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:11:02.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:11:02.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:11:02.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:11:02.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:11:02.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:11:02.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:11:02.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:11:02.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:02.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:11:02.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:11:02.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:02.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:11:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:11:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:11:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:11:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:11:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:11:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:11:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:11:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:11:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:11:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:11:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:11:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:11:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:11:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:11:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:11:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:11:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:11:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:11:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:11:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:11:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:11:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:11:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:11:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:11:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:11:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:11:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:11:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:11:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:11:02.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:11:02.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:11:02.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:11:02.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:11:02.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:11:02.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:02.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:11:02.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:02.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:11:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:11:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:11:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:11:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:11:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:11:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:11:03.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:11:03.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:10:58 2026-02-20T21:11:03.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:11:03.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:11:03.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:11:03.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:11:03.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:01' 2026-02-20T21:11:03.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:11:03.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 1 2026-02-20T21:11:03.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:03.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:11:05.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:11:05.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:05.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:05.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:05.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:05.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:11:05.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:11:05.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:11:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:11:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:11:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:11:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:03 2026-02-20T21:11:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:11:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:11:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:11:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:11:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:01' 2026-02-20T21:11:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:11:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:11:05.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:11:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:11:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:11:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:11:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:11:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:11:05.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:11:05.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:11:05.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:11:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:11:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:03' 2026-02-20T21:11:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:11:03' 2026-02-20T21:11:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:11:03' 2026-02-20T21:11:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:05.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:11:05.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:11:03cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:11:01' 2026-02-20T21:11:05.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:11:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:11:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:03' 2026-02-20T21:11:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:11:05.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:11:05.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:11:05.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:11:05.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:11:05.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:05.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:05.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:11:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:11:06.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:11:06.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:06.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:06.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:06.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:06.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:06.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:06.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:06.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:06.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:11:06.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:11:06.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:11:06.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:11:06.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:11:06.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:11:06.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:01 2026-02-20T21:11:06.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:11:06.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:11:06.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:11:06.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:11:06.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:03' 2026-02-20T21:11:06.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:11:06.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:11:06.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:11:06.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:11:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:11:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:11:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:11:06.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:11:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:11:06.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:11:06.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:11:06.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:11:06.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:01' 2026-02-20T21:11:06.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:11:01' 2026-02-20T21:11:06.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:06.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:06.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:11:01' 2026-02-20T21:11:06.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:06.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:06.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:11:06.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:11:01cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:11:03' 2026-02-20T21:11:06.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:11:06.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:11:06.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:01' 2026-02-20T21:11:06.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:11:06.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:11:06.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:11:06.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:11:06.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:11:06.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:06.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:06.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster2 mirror test 2026-02-20T21:11:06.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:06.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:06.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:06.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:11:06.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-20T21:11:08.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:11:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:11:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:11:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:08.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:08.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:08.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:11:08.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:11:08.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:08.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:11:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:11:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:11:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:11:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:11:09.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:11:09.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:11:09.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:11:09.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:09.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:11:09.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:11:09.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:11:09.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:09.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:11:09.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:09.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:11:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:11:09.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:09.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:09.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:09.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:09.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:11:09.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:11:09.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:11:09.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:11:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:11:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:11:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:11:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:11:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:11:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:11:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:11:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:11:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:11:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:11:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:11:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:09.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:09.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:11:13.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 72 2026-02-20T21:11:13.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:11:13.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:11:13.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:13.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:11:13.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:11:13.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:13.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:13.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:11:13.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:13.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:11:13.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:11:13.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:11:13.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:13.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:11:13.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:11:13.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:13.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:13.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:13.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:11:13.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:11:13.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:11:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:11:13.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=72 2026-02-20T21:11:13.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:11:13.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:13.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:11:13.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:11:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:11:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:11:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:11:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:11:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=72 2026-02-20T21:11:13.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 72 = 72 2026-02-20T21:11:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-20T21:11:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:11:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:11:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-20T21:11:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:11:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:14.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:14.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:14.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:11:14.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:11:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:11:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:11:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:11:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:11:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:11 2026-02-20T21:11:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:11:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:11:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:11:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621867,"replay_state":"idle"} 2026-02-20T21:11:14.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:08 2026-02-20T21:11:14.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:11:14.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 72 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.3f289b0f-761f-443f-aa89-ff6b90859b01 (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:11:14.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:11:14.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:11:14.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:11:14.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:11:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:11:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:11:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:11:14.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:11:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:11:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:11:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:11:14.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:11:14.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:11' 2026-02-20T21:11:14.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-20 21:11:11' 2026-02-20T21:11:14.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:14.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:14.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:14.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:11:14.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:11:11' 2026-02-20T21:11:14.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:14.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:14.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:11:14.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:11:11cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621867,"replay_state":"idle"}2026-02-20 21:11:08' 2026-02-20T21:11:14.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:11:14.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:11:14.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:11' 2026-02-20T21:11:14.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:11:14.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:11:14.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:11:14.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:11:14.433 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:11:14.434 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:14.434 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:14.434 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:11:14.434 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:14.434 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:14.434 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:14.434 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:11:14.434 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:14.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:14.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:14.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:11:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:11:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:11:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:11:15.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:11:15.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:11:15.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:11:15.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621867,"replay_state":"idle"} 2026-02-20T21:11:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:11:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:08 2026-02-20T21:11:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:11:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:11:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:11:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:11:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:11' 2026-02-20T21:11:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:11:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:11:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:11:15.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621867,"replay_state":"idle"} 2026-02-20T21:11:15.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:11:15.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:11:15.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:11:15.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:11:15.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:11:15.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:11:15.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:11:15.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:11:15.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:08' 2026-02-20T21:11:15.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:11:08' 2026-02-20T21:11:15.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:15.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:15.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:15.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:11:15.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:11:08' 2026-02-20T21:11:15.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:15.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:15.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:11:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621867,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:11:08cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:11:11' 2026-02-20T21:11:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:11:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:11:15.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:08' 2026-02-20T21:11:15.507 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:11:15.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:11:15.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:11:15.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:11:15.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:11:15.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:15.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:15.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:11:15.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test 2026-02-20T21:11:15.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:15.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:15.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:15.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-20T21:11:16.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:11:16.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:11:16.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:16.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:16.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:16.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:11:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:11:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:11:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:11:17.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:11:17.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:17.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:17.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:17.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:11:17.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:11:17.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:11:17.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:11:17.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:11:17.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:11:17.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:17.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:11:17.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:11:17.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:11:17.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:17.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:11:17.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:17.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:11:17.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:17.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:11:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:11:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:11:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:11:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:11:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:11:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:11:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:11:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:11:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:11:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:11:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:11:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:11:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:11:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:11:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:11:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:11:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:11:17.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:11:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:11:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:11:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:11:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:11:17.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:11:17.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:11:17.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:11:17.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:11:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:11:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:11:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:11:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:17.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:11:19.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:11:19.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:19.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:19.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:19.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:11:19.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:11:19.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:11:19.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:11:19.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:11:19.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:19.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:11:19.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:11:19.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:11:19.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:19.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:11:19.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:11:19.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:19.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:11:19.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:19.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:11:19.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:11:19.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:19.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:11:19.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:19.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:11:19.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:11:19.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:11:19.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:11:19.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:11:19.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:19.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:11:19.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:11:19.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:11:19.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:19.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:19.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:11:19.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:11:19.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:11:19.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:19.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:11:19.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:11:19.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:11:19.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:11:19.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:19.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:19.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:11:19.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:11:19.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:11:19.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:11:19.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:19.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:19.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:19.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:19.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:11:19.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:11:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:11:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:11:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:11:19.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:11:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:11:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:19.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:11:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:11:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:20.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:11:20.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:11:20.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:11:20.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:11:20.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:11:20.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:11:20.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:18 2026-02-20T21:11:20.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:11:20.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:11:20.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:11:20.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:11:20.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:16' 2026-02-20T21:11:20.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:11:20.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:11:20.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:11:20.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:11:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:11:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:11:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:11:20.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:11:20.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:11:20.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:11:20.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:11:20.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:11:20.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:18' 2026-02-20T21:11:20.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:11:18' 2026-02-20T21:11:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:11:18' 2026-02-20T21:11:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:11:20.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:11:18cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:11:16' 2026-02-20T21:11:20.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:11:20.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:11:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:18' 2026-02-20T21:11:20.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:11:20.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:11:20.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:11:20.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:11:20.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:11:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:11:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:20.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:11:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:11:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:11:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:11:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:11:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:11:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:11:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:11:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:16 2026-02-20T21:11:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:11:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:11:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:11:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:11:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:18' 2026-02-20T21:11:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:11:21.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:11:21.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:11:21.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:11:21.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:11:21.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:11:21.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:11:21.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:11:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:11:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:11:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:11:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:11:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:16' 2026-02-20T21:11:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:11:16' 2026-02-20T21:11:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:11:16' 2026-02-20T21:11:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:11:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:11:16cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:11:18' 2026-02-20T21:11:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:11:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:11:21.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:16' 2026-02-20T21:11:21.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:11:21.401 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:11:21.401 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:11:21.401 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:11:21.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:11:21.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:21.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:21.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test 2026-02-20T21:11:21.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:21.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:21.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:21.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:11:21.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-20T21:11:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:11:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-20T21:11:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-20T21:11:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:11:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:11:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:11:23.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:11:23.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:23.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:11:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:11:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:11:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:11:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:11:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:11:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:11:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:11:23.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:11:23.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:23.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:11:23.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:11:23.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:11:23.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:23.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:11:23.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:11:23.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:23.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:11:23.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:11:23.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:11:23.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:11:23.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:11:23.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:11:23.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:11:23.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:11:23.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:11:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:11:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:11:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:11:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:11:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:11:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:11:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:11:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:11:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:11:23.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:23.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:23.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:11:23.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:11:23.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:11:23.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:11:23.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:11:23.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:11:23.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:11:23.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:11:23.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:11:23.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:11:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:11:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:11:23.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:11:23.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:23.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:11:23.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:11:23.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:23.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:11:23.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:11:23.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:11:23.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:11:23.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:11:23.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:11:23.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:11:23.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:11:23.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:11:23.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:23.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:11:23.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:11:23.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:11:23.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:11:23.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:11:23.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:11:23.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:11:23.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:11:23.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:11:23.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:11:23.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:23.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:11:23.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:11:23.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:11:23.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:11:23.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:11:23.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:11:23.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:11:24.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:11:24.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:11:24.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:11:24.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:11:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:11:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:11:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:11:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:11:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:11:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:11:26.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:11:26.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:26.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:26.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:26.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:11:26.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:11:26.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:11:26.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:11:26.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:11:26.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:11:26.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:26.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:11:26.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:11:26.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:11:26.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:26.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:11:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:11:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:11:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:11:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:11:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:11:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:11:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:11:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:11:26.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:11:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:11:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:11:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:11:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:11:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:11:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:11:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:11:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:11:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:11:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:11:26.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:11:26.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:11:26.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:11:26.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:26.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:26.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:11:26.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:11:26.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:11:26.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:11:26.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:11:26.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:11:26.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:11:26.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:11:26.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:11:26.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:11:26.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:11:26.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:11:26.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:11:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:11:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:11:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:11:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:11:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:11:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:11:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:11:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:11:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:11:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:11:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:11:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:11:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:11:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:11:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:11:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:11:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:11:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:11:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:11:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:11:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:11:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:11:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:11:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:11:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:11:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:11:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:11:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:11:26.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:11:26.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:11:26.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:11:26.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:11:26.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:11:26.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:11:26.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:11:26.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:11:26.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:11:26.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:26.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:11:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:11:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:11:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:11:30.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:11:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:11:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:11:30.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:11:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:11:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:11:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:11:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:11:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:11:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:11:30.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:30.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:11:30.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:11:30.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:30.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:11:30.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:11:30.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:11:30.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:11:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:11:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:11:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:11:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:11:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:11:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:11:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:11:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:11:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:11:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:11:30.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:11:30.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:11:30.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:11:30.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:11:30.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:30.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:30.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:11:30.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:11:30.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:11:30.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:11:30.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:11:30.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:11:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:11:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:11:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:11:30.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:11:30.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:11:30.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:11:30.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:11:30.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:30.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:11:30.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:11:30.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:30.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:11:30.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:11:30.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:11:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:11:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:11:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:11:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:11:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:11:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:11:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:11:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:11:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:11:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:11:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:11:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:11:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:11:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:11:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:11:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:11:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:11:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:11:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:11:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:11:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:11:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:11:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:11:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:11:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:11:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:11:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:11:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:11:30.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:11:30.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:11:30.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:11:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:11:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:11:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:11:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:11:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:11:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:11:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:11:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:11:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:11:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-20T21:11:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:30.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:30.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-20T21:11:33.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 75 2026-02-20T21:11:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:11:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:11:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:11:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:11:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:11:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:11:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:33.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:11:33.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:33.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:11:33.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:11:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:11:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:11:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:11:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:11:33.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:11:33.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=75 2026-02-20T21:11:33.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:11:33.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:33.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:11:34.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:11:34.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:34.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:34.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:34.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:11:34.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:11:34.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:11:34.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:11:34.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=75 2026-02-20T21:11:34.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 75 = 75 2026-02-20T21:11:34.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:34.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:11:34.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:34.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:34.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:34.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:11:34.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:34.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:34.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:34.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:11:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:11:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:11:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:11:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:11:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:11:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:11:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:11:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:11:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:33 2026-02-20T21:11:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:11:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:11:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:11:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621881,"replay_state":"idle"} 2026-02-20T21:11:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:26 2026-02-20T21:11:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:11:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 75 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.19343876-a148-42cc-a97d-d59f0b607fc2 (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:11:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:11:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:11:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:11:35.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:11:35.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:11:35.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:11:35.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:11:35.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:11:35.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:11:35.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:11:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:11:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:11:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:33' 2026-02-20T21:11:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:11:33' 2026-02-20T21:11:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:11:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:11:33' 2026-02-20T21:11:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:35.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:11:35.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary44044409mirror.0trial1992026-02-20 21:11:33cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621881,"replay_state":"idle"}2026-02-20 21:11:26' 2026-02-20T21:11:35.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:11:35.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:11:35.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:33' 2026-02-20T21:11:35.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:11:35.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:11:35.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:11:35.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:11:35.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:11:35.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:35.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:35.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-20T21:11:35.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:35.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:35.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:35.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:11:35.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:35.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:35.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:35.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:11:36.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-20T21:11:36.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:36.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:36.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:36.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:11:36.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:36.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:36.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:36.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:36.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:11:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:11:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:11:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:11:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621881,"replay_state":"idle"} 2026-02-20T21:11:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:11:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:26 2026-02-20T21:11:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:11:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:11:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:11:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:11:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:33' 2026-02-20T21:11:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:11:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:11:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:11:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621881,"replay_state":"idle"} 2026-02-20T21:11:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:11:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:11:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:11:36.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:11:36.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:11:36.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:11:36.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:11:36.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:11:36.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:26' 2026-02-20T21:11:36.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-20 21:11:26' 2026-02-20T21:11:36.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:36.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:36.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:36.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:11:36.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:11:26' 2026-02-20T21:11:36.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:36.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:36.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:11:36.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621881,"replay_state":"idle"}45294534mirror.1trial1992026-02-20 21:11:26cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:11:33' 2026-02-20T21:11:36.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:11:36.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:11:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:26' 2026-02-20T21:11:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:11:36.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:11:36.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:11:36.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:11:36.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:11:36.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster1 mirror test 2026-02-20T21:11:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-20T21:11:36.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:11:36.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-20T21:11:36.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:36.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:36.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:36.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-20T21:11:36.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:36.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:36.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:36.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:11:36.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:11:36.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:36.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:11:37.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:11:37.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:37.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:37.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:37.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:11:37.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:11:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:11:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:11:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:11:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:11:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:11:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:11:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:11:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:11:37.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:11:37.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:11:37.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:11:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:11:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:11:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:11:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:11:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:11:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:11:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:11:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:11:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:11:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:11:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:11:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:11:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:11:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:11:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:11:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:11:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:11:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:11:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:11:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:11:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:11:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:11:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:11:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:11:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:11:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:11:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:11:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:11:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:11:37.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:11:37.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:11:37.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:11:37.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:37.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:11:37.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:11:37.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:37.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:11:37.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:11:37.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:11:37.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:11:37.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:11:37.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:11:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:11:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:11:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:11:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:11:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:11:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:11:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:11:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:11:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:11:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:11:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:11:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:11:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:11:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:11:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:11:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:11:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:11:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:11:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:11:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:11:37.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:11:37.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:11:37.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:11:37.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:11:38.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:11:38.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:11:38.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:11:38.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:11:38.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:11:38.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:11:38.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:11:38.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:38.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:11:40.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:11:40.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:40.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:40.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:40.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:11:40.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:11:40.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:11:40.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:11:40.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:11:40.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:11:40.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:40.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:11:40.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:11:40.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:11:40.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:40.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:11:40.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:11:40.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:11:40.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:40.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:11:40.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:11:40.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:40.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:11:40.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:11:40.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:11:40.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:11:40.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:11:40.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:11:40.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:11:40.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:11:40.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:11:40.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:40.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:11:40.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:40.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:11:40.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:11:40.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:11:40.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:11:40.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:11:40.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:11:40.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:11:40.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:11:40.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:40.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:40.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:11:40.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:11:40.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:11:40.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:11:40.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:11:40.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:11:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:11:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:11:40.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:11:40.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:11:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:11:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:11:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:11:40.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:40.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:11:40.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:11:40.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:40.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:11:40.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:11:40.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:11:40.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:11:40.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:11:40.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:11:40.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:11:40.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:11:40.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:11:40.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:40.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:11:40.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:11:40.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:11:40.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:11:40.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:11:40.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:11:40.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:11:40.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:11:40.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:11:40.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:11:40.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:40.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:11:40.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:11:40.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:11:40.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:11:40.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:11:40.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:11:40.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:11:40.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:11:40.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:11:40.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:11:40.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:11:40.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:11:40.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:11:40.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:11:40.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:11:40.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:11:40.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:40.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:11:40.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:40.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:40.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:40.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:11:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:11:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:41.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:41.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:11:41.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:11:41.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:11:41.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:11:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:11:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:11:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:38 2026-02-20T21:11:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:11:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:11:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:11:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:11:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:38' 2026-02-20T21:11:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:11:41.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:11:41.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:11:41.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:11:41.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:11:41.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:11:41.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:11:41.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:11:41.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:11:41.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:11:41.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:11:41.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:11:41.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:38' 2026-02-20T21:11:41.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:11:38' 2026-02-20T21:11:41.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:41.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:41.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:41.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:41.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:11:38' 2026-02-20T21:11:41.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:41.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:41.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:11:41.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:11:38cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image demoted2026-02-20 21:11:38' 2026-02-20T21:11:41.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:11:41.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:11:41.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:38' 2026-02-20T21:11:41.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:11:41.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:11:41.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:11:41.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:11:41.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:11:41.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:41.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:41.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:11:41.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:41.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:41.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:41.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:41.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:41.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:41.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:41.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:11:42.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:11:42.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:42.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:42.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:42.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:42.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:42.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:42.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:42.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:42.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:11:42.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:11:42.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:11:42.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:11:42.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:11:42.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:11:42.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:41 2026-02-20T21:11:42.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:11:42.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:11:42.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:11:42.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:11:42.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:38' 2026-02-20T21:11:42.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:11:42.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:11:42.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:11:42.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:11:42.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:11:42.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:11:42.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:11:42.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:11:42.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:11:42.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:11:42.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:11:42.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:11:42.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:41' 2026-02-20T21:11:42.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:11:41' 2026-02-20T21:11:42.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:42.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:42.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:42.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:42.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:11:41' 2026-02-20T21:11:42.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:42.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:42.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:11:42.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:11:41cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:11:38' 2026-02-20T21:11:42.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:11:42.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:11:42.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:41' 2026-02-20T21:11:42.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:11:42.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:11:42.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:11:42.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:11:42.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:11:42.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:42.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:42.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster2 mirror test 2026-02-20T21:11:42.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:42.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:42.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:42.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:11:42.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-20T21:11:43.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:11:43.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:11:43.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:43.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:43.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:43.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:11:43.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:43.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:43.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:43.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:11:43.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:11:43.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:43.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:11:44.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:11:44.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:44.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:44.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:44.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:11:44.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:11:44.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:11:44.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:11:44.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:11:44.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:11:44.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:44.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:11:44.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:11:44.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:11:44.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:44.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:11:44.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:44.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:11:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:11:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:11:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:11:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:11:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:11:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:11:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:11:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:11:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:11:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:11:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:11:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:11:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:11:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:11:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:11:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:11:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:11:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:11:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:11:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:11:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:11:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:11:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:44.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:44.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:44.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:44.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:11:44.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:11:44.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:11:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:11:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:11:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:11:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:11:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:11:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:11:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:11:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:11:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:11:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:11:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:11:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:11:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:11:48.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 78 2026-02-20T21:11:48.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:11:48.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:11:48.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:48.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:11:48.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:11:48.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:48.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:48.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:11:48.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:48.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:11:48.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:11:48.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:11:48.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:48.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:11:48.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:11:48.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:48.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:48.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:48.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:11:48.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:11:48.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:11:48.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:11:48.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=78 2026-02-20T21:11:48.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:11:48.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:48.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:11:48.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:11:48.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:48.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:48.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:48.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:11:48.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:11:48.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:11:48.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:11:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=78 2026-02-20T21:11:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 78 = 78 2026-02-20T21:11:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-20T21:11:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:11:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:11:49.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-20T21:11:49.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:49.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:49.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:49.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:11:49.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:49.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:49.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:49.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:49.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:11:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:11:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:11:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:11:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:11:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:11:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:46 2026-02-20T21:11:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:11:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:11:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:11:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621902,"replay_state":"idle"} 2026-02-20T21:11:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:43 2026-02-20T21:11:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:11:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 78 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.cc5d12b7-4d7f-40d8-8f79-0b5e78d1eef2 (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:11:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:11:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:11:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:11:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:11:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:11:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:11:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:11:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:11:49.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:11:49.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:11:49.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:11:49.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:11:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:46' 2026-02-20T21:11:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-20 21:11:46' 2026-02-20T21:11:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:11:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:11:46' 2026-02-20T21:11:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:11:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:11:46cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621902,"replay_state":"idle"}2026-02-20 21:11:43' 2026-02-20T21:11:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:11:49.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:11:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:46' 2026-02-20T21:11:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:11:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:11:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:11:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:11:49.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:11:49.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:49.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:49.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:11:49.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:49.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:49.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:49.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:11:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:11:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:11:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:11:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:11:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:11:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:11:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:11:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621902,"replay_state":"idle"} 2026-02-20T21:11:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:11:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:43 2026-02-20T21:11:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:11:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:11:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:11:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:11:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:46' 2026-02-20T21:11:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:11:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:11:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:11:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621902,"replay_state":"idle"} 2026-02-20T21:11:50.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:11:50.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:11:50.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:11:50.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:11:50.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:11:50.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:11:50.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:11:50.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:11:50.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:43' 2026-02-20T21:11:50.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:11:43' 2026-02-20T21:11:50.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:50.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:50.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:50.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:11:50.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:11:43' 2026-02-20T21:11:50.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:50.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:50.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:11:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621902,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:11:43cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:11:46' 2026-02-20T21:11:50.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:11:50.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:11:50.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:43' 2026-02-20T21:11:50.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:11:50.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:11:50.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:11:50.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:11:50.512 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:11:50.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:50.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:50.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:11:50.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test 2026-02-20T21:11:50.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:50.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:50.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:50.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-20T21:11:51.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:11:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:11:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:11:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:11:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:11:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:11:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:11:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:11:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:11:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:11:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:11:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:11:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:11:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:11:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:11:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:11:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:11:52.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:52.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:11:52.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:52.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:11:52.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:11:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:11:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:11:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:11:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:11:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:11:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:11:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:11:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:11:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:11:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:11:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:11:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:11:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:11:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:11:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:11:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:11:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:11:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:11:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:11:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:11:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:52.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:52.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:11:52.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:11:52.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:11:52.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:11:52.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:11:52.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:11:52.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:11:52.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:11:52.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:52.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:11:54.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:11:54.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:54.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:54.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:54.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:11:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:11:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:11:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:11:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:11:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:11:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:11:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:11:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:11:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:11:54.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:54.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:11:54.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:54.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:11:54.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:11:54.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:54.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:11:54.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:54.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:11:54.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:11:54.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:11:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:11:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:11:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:11:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:11:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:11:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:11:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:11:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:11:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:11:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:11:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:11:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:11:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:11:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:11:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:11:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:11:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:54.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:54.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:54.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:54.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:11:54.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:11:54.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:11:54.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:11:54.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:11:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:11:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:11:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:11:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:11:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:11:55.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:11:55.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:11:55.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:11:55.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:11:55.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:11:55.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:53 2026-02-20T21:11:55.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:11:55.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:11:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:11:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:11:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:51' 2026-02-20T21:11:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:11:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:11:55.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:11:55.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:11:55.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:11:55.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:11:55.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:11:55.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:11:55.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:11:55.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:11:55.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:11:55.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:11:55.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:53' 2026-02-20T21:11:55.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:11:53' 2026-02-20T21:11:55.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:55.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:55.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:55.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:55.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:11:53' 2026-02-20T21:11:55.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:55.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:55.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:11:55.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:11:53cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:11:51' 2026-02-20T21:11:55.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:11:55.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:11:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:53' 2026-02-20T21:11:55.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:11:55.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:11:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:11:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:11:55.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:11:55.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:55.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:55.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:11:55.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:55.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:55.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:55.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:55.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:55.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:55.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:55.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:11:56.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:11:56.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:56.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:56.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:56.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:56.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:11:56.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:11:56.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:56.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:56.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:11:56.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:11:56.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:11:56.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:11:56.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:11:56.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:11:56.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:51 2026-02-20T21:11:56.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:11:56.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:11:56.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:11:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:11:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:11:53' 2026-02-20T21:11:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:11:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:11:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:11:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:11:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:11:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:11:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:11:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:11:56.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:11:56.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:11:56.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:11:56.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:11:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:51' 2026-02-20T21:11:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:11:51' 2026-02-20T21:11:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:11:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:11:51' 2026-02-20T21:11:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:11:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:11:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:11:56.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:11:51cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:11:53' 2026-02-20T21:11:56.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:11:56.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:11:56.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:11:51' 2026-02-20T21:11:56.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:11:56.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:11:56.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:11:56.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:11:56.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:11:56.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:56.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:11:56.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test 2026-02-20T21:11:56.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:11:56.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:56.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:56.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:11:56.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-20T21:11:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:11:57.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-20T21:11:57.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:57.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:57.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:57.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-20T21:11:57.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:57.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:57.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:57.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:11:57.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:11:57.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:57.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:11:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:11:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:11:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:11:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:11:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:11:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:11:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:11:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:11:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:11:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:11:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:11:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:11:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:11:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:11:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:11:58.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:11:58.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:11:58.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:11:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:11:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:11:58.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:11:58.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:11:58.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:11:58.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:11:58.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:11:58.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:11:58.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:11:58.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:11:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:11:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:11:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:11:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:11:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:11:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:11:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:11:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:11:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:11:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:11:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:11:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:11:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:11:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:11:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:11:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:11:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:11:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:11:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:11:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:11:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:11:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:11:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:11:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:11:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:11:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:11:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:11:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:11:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:11:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:11:58.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:11:58.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:11:58.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:11:58.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:11:59.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:11:59.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:11:59.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:11:59.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:11:59.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:11:59.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:59.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:12:01.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:12:01.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:01.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:01.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:01.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:12:01.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:12:01.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:12:01.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:12:01.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:12:01.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:01.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:12:01.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:12:01.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:12:01.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:12:01.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:01.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:12:01.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:12:01.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:12:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:12:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:12:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:12:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:12:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:12:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:12:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:12:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:12:01.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:12:01.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:12:01.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:12:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:12:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:12:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:12:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:12:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:12:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:12:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:12:01.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:12:01.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:12:01.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:12:01.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:12:01.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:01.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:12:01.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:12:01.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:12:01.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:12:01.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:12:01.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:12:01.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:12:01.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:12:01.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:12:01.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:12:01.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:01.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:12:01.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:12:01.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:12:01.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:12:01.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:12:01.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:12:01.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:12:01.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:01.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:01.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:01.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:12:01.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:12:01.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:12:01.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:12:01.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:12:01.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:12:01.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:01.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:12:05.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:12:05.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:05.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:12:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:12:05.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:12:05.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:12:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:12:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:12:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:12:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:12:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:12:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:12:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:12:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:12:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:12:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:12:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:12:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:12:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:12:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:12:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:12:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:12:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:12:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:12:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:12:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:12:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:12:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:12:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:12:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:12:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:12:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:12:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:12:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:12:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:12:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:12:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:12:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:12:05.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:12:05.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:12:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:12:05.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:12:05.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:12:05.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:12:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:12:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:12:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:12:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:12:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:12:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:12:05.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:12:05.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:12:05.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:12:05.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:12:05.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:12:05.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:05.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:12:05.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:12:05.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:12:05.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:12:05.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:12:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:12:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:12:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:12:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:12:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:12:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:12:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:12:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:12:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:12:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:12:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:12:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:12:05.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:05.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:05.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:05.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:12:05.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:12:05.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:12:05.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:12:05.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:12:05.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:12:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:12:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:12:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:12:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:12:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:12:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:12:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:12:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:12:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-20T21:12:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-20T21:12:08.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 81 2026-02-20T21:12:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:12:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:12:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:12:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:12:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:12:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:12:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:12:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:12:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:08.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:12:08.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:12:08.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:08.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:08.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:08.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:12:08.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:12:08.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:12:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:12:08.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=81 2026-02-20T21:12:08.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:12:08.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:08.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:12:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:12:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:12:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:12:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:12:09.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:12:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=81 2026-02-20T21:12:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 81 = 81 2026-02-20T21:12:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:12:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:09.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:12:09.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:09.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:09.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:09.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:12:10.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:12:10.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:10.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:10.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:10.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:12:10.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:10.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:10.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:10.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:10.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:12:10.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:12:10.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:12:10.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:12:10.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:12:10.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:12:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:08 2026-02-20T21:12:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:12:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:12:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:12:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621916,"replay_state":"idle"} 2026-02-20T21:12:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:01 2026-02-20T21:12:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:12:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 81 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.a35b8250-2e20-47c0-9a7c-e0a4102e69e2 (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:12:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:12:10.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:12:10.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:12:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:12:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:12:10.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:12:10.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:12:10.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:12:10.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:12:10.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:12:10.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:12:10.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:12:10.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:08' 2026-02-20T21:12:10.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:12:08' 2026-02-20T21:12:10.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:10.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:10.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:10.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:12:10.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:12:08' 2026-02-20T21:12:10.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:10.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:10.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:12:10.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary44044409mirror.0trial1992026-02-20 21:12:08cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621916,"replay_state":"idle"}2026-02-20 21:12:01' 2026-02-20T21:12:10.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:12:10.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:12:10.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:08' 2026-02-20T21:12:10.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:12:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:12:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:12:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:12:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:12:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-20T21:12:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:12:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:10.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:12:11.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-20T21:12:11.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:11.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:11.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:11.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:12:11.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:11.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:12:11.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:12:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:12:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:12:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621916,"replay_state":"idle"} 2026-02-20T21:12:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:12:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:01 2026-02-20T21:12:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:12:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:12:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:12:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:12:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:08' 2026-02-20T21:12:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:12:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:12:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:12:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621916,"replay_state":"idle"} 2026-02-20T21:12:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:12:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:12:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:12:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:12:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:12:11.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:12:11.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:12:11.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:12:11.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:01' 2026-02-20T21:12:11.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-20 21:12:01' 2026-02-20T21:12:11.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:11.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:11.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:11.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:12:11.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:12:01' 2026-02-20T21:12:11.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:11.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:11.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:12:11.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621916,"replay_state":"idle"}45294534mirror.1trial1992026-02-20 21:12:01cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:12:08' 2026-02-20T21:12:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:12:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:12:11.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:01' 2026-02-20T21:12:11.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:12:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:12:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:12:11.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:12:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:12:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:11.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:11.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster1 mirror test 2026-02-20T21:12:11.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:11.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:11.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:11.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-20T21:12:11.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:12:11.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-20T21:12:11.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:11.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:11.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:11.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-20T21:12:11.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:11.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:11.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:11.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:12:11.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:12:11.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:11.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:12:12.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:12:12.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:12.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:12.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:12.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:12:12.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:12:12.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:12:12.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:12:12.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:12:12.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:12:12.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:12.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:12:12.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:12:12.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:12:12.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:12.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:12:12.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:12:12.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:12:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:12:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:12:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:12:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:12:12.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:12:12.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:12:12.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:12:12.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:12:12.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:12:12.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:12:12.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:12:12.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:12.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:12:12.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:12.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:12:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:12:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:12:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:12:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:12:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:12:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:12:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:12:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:12:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:12:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:12:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:12:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:12:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:12:12.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:12.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:12.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:12.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:12:12.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:12:12.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:12:12.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:12:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:12:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:12:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:12:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:12:12.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:12:12.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:12:12.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:12:12.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:12:12.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:12:12.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:12:12.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:12:12.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:12.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:12:12.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:12:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:12:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:12:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:12:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:12:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:12:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:12:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:12:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:12:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:12:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:12:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:12:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:12:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:12:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:12:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:12:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:13.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:12:13.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:12:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:12:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:12:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:12:13.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:12:13.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:12:13.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:12:13.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:13.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:12:15.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:12:15.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:15.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:15.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:15.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:12:15.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:12:15.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:12:15.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:12:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:12:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:12:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:12:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:12:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:12:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:12:15.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:12:15.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:12:15.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:15.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:12:15.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:12:15.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:15.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:12:15.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:12:15.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:12:15.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:12:15.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:12:15.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:12:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:12:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:12:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:12:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:12:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:12:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:12:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:12:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:12:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:12:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:12:15.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:12:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:12:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:12:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:12:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:12:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:12:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:12:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:12:15.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:15.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:15.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:15.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:12:15.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:12:15.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:12:15.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:12:15.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:15.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:12:15.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:12:15.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:15.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:12:15.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:12:15.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:12:15.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:12:15.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:12:15.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:12:15.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:12:15.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:12:15.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:12:15.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:15.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:12:15.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:12:15.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:12:15.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:12:15.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:12:15.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:12:15.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:12:15.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:12:15.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:12:15.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:12:15.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:15.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:12:15.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:12:15.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:12:15.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:12:15.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:12:15.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:12:15.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:12:15.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:15.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:15.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:15.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:12:15.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:12:15.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:12:15.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:12:15.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:12:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:12:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:12:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:12:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:12:16.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:12:16.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:16.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:16.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:16.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:12:16.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:16.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:16.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:16.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:16.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:12:16.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:12:16.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:12:16.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:12:16.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:12:16.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:12:16.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:13 2026-02-20T21:12:16.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:12:16.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:12:16.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:12:16.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:12:16.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:13' 2026-02-20T21:12:16.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:12:16.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:12:16.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:12:16.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:12:16.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:12:16.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:12:16.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:12:16.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:12:16.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:12:16.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:12:16.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:12:16.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:12:16.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:13' 2026-02-20T21:12:16.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:12:13' 2026-02-20T21:12:16.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:16.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:16.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:16.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:12:16.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:12:13' 2026-02-20T21:12:16.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:16.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:16.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:12:16.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:12:13cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image demoted2026-02-20 21:12:13' 2026-02-20T21:12:16.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:12:16.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:12:16.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:13' 2026-02-20T21:12:16.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:12:16.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:12:16.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:12:16.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:12:16.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:12:16.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:16.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:16.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:12:16.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:16.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:16.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:16.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:12:16.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:16.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:16.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:16.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:12:17.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:12:17.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:17.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:17.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:17.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:12:17.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:17.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:17.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:17.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:17.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:12:17.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:12:17.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:12:17.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:12:17.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:12:17.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:12:17.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:16 2026-02-20T21:12:17.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:12:17.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:12:17.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:12:17.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:12:17.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:13' 2026-02-20T21:12:17.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:12:17.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:12:17.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:12:17.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:12:17.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:12:17.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:12:17.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:12:17.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:12:17.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:12:17.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:12:17.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:12:17.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:12:17.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:16' 2026-02-20T21:12:17.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:12:16' 2026-02-20T21:12:17.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:17.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:17.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:17.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:12:17.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:12:16' 2026-02-20T21:12:17.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:17.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:17.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:12:17.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:12:16cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:12:13' 2026-02-20T21:12:17.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:12:17.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:12:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:16' 2026-02-20T21:12:17.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:12:17.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:12:17.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:12:17.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:12:17.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:12:17.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:17.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:17.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster2 mirror test 2026-02-20T21:12:17.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:17.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:17.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:17.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:12:17.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-20T21:12:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:12:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:12:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:12:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:18.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:12:18.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:12:18.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:18.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:12:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:12:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:12:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:12:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:12:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:12:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:12:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:12:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:12:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:12:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:12:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:12:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:12:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:12:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:12:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:19.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:12:19.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:19.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:12:19.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:12:19.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:12:19.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:12:19.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:12:19.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:19.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:12:19.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:12:19.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:12:19.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:19.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:19.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:12:19.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:12:19.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:12:19.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:19.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:12:19.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:12:19.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:12:19.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:12:19.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:19.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:12:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:12:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:12:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:12:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:19.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:19.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:19.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:12:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:12:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:12:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:12:19.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:12:19.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:12:19.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:12:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:12:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:12:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:12:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:12:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:12:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:12:19.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:12:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:12:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:12:22.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 84 2026-02-20T21:12:22.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:12:22.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:12:22.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:22.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:12:22.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:12:22.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:22.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:22.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:12:22.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:22.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:12:22.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:12:22.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:12:22.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:22.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:12:22.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:12:22.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:22.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:22.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:22.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:12:22.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:12:22.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:12:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:12:22.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=84 2026-02-20T21:12:22.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:12:22.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:22.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:12:23.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:12:23.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:23.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:23.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:23.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:12:23.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:12:23.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:12:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:12:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=84 2026-02-20T21:12:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 84 = 84 2026-02-20T21:12:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-20T21:12:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:12:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:23.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:12:24.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-20T21:12:24.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:24.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:24.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:24.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:12:24.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:24.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:24.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:24.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:24.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:12:24.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:12:24.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:12:24.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:12:24.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:12:24.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:12:24.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:21 2026-02-20T21:12:24.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:12:24.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:12:24.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:12:24.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621937,"replay_state":"idle"} 2026-02-20T21:12:24.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:18 2026-02-20T21:12:24.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:12:24.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 84 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.7909205a-ba02-4a3f-8343-1c2163ba75fd (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:12:24.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:12:24.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:12:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:12:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:12:24.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:12:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:12:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:12:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:12:24.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:12:24.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:12:24.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:12:24.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:12:24.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:21' 2026-02-20T21:12:24.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-20 21:12:21' 2026-02-20T21:12:24.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:24.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:24.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:24.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:12:24.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:12:21' 2026-02-20T21:12:24.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:24.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:24.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:12:24.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:12:21cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621937,"replay_state":"idle"}2026-02-20 21:12:18' 2026-02-20T21:12:24.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:12:24.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:12:24.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:21' 2026-02-20T21:12:24.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:12:24.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:12:24.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:12:24.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:12:24.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:12:24.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:24.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:24.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:12:24.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:24.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:24.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:24.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:12:24.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:24.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:24.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:24.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:12:25.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:12:25.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:25.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:25.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:25.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:12:25.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:25.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:25.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:25.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:25.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:12:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:12:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:12:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:12:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621937,"replay_state":"idle"} 2026-02-20T21:12:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:12:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:18 2026-02-20T21:12:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:12:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:12:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:12:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:12:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:21' 2026-02-20T21:12:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:12:25.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:12:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:12:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621937,"replay_state":"idle"} 2026-02-20T21:12:25.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:12:25.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:12:25.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:12:25.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:12:25.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:12:25.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:12:25.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:12:25.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:12:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:18' 2026-02-20T21:12:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:12:18' 2026-02-20T21:12:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:12:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:12:18' 2026-02-20T21:12:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:12:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621937,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:12:18cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:12:21' 2026-02-20T21:12:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:12:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:12:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:18' 2026-02-20T21:12:25.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:12:25.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:12:25.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:12:25.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:12:25.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:12:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:12:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test 2026-02-20T21:12:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:25.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:25.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:25.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-20T21:12:26.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:12:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:12:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:12:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:12:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:12:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:12:27.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:12:27.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:27.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:27.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:27.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:12:27.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:12:27.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:12:27.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:12:27.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:12:27.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:27.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:12:27.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:12:27.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:12:27.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:12:27.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:27.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:12:27.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:27.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:12:27.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:27.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:12:27.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:12:27.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:27.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:12:27.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:27.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:12:27.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:12:27.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:12:27.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:12:27.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:12:27.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:27.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:12:27.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:12:27.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:12:27.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:27.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:27.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:12:27.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:12:27.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:12:27.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:27.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:12:27.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:12:27.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:12:27.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:12:27.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:27.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:27.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:12:27.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:12:27.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:12:27.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:12:27.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:27.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:27.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:27.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:27.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:12:27.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:12:27.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:12:27.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:12:27.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:12:27.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:12:27.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:12:27.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:12:27.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:27.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:12:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:12:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:12:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:12:29.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:12:29.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:12:29.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:12:29.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:12:29.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:29.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:12:29.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:12:29.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:12:29.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:29.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:12:29.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:29.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:12:29.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:29.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:12:29.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:12:29.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:29.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:12:29.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:29.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:12:29.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:12:29.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:12:29.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:12:29.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:12:29.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:29.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:12:29.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:12:29.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:12:29.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:29.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:29.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:12:29.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:12:29.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:12:29.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:29.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:12:29.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:12:29.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:12:29.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:12:29.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:29.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:29.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:12:29.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:12:29.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:12:29.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:12:29.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:29.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:29.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:29.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:29.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:12:29.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:12:29.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:12:29.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:12:29.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:12:29.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:12:29.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:29.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:12:29.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:29.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:29.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:29.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:12:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:12:30.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:12:30.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:30.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:30.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:30.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:12:30.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:30.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:30.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:30.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:30.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:12:30.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:12:30.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:12:30.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:12:30.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:12:30.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:12:30.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:28 2026-02-20T21:12:30.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:12:30.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:12:30.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:12:30.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:12:30.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:26' 2026-02-20T21:12:30.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:12:30.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:12:30.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:12:30.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:12:30.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:12:30.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:12:30.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:12:30.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:12:30.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:12:30.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:12:30.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:12:30.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:12:30.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:28' 2026-02-20T21:12:30.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:12:28' 2026-02-20T21:12:30.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:30.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:30.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:30.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:12:30.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:12:28' 2026-02-20T21:12:30.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:30.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:30.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:12:30.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:12:28cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:12:26' 2026-02-20T21:12:30.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:12:30.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:12:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:28' 2026-02-20T21:12:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:12:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:12:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:12:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:12:30.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:12:30.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:30.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:30.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:12:30.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:30.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:30.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:30.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:12:30.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:30.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:30.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:30.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:12:31.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:12:31.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:31.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:31.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:31.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:12:31.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:31.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:31.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:31.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:31.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:12:31.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:12:31.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:12:31.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:12:31.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:12:31.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:12:31.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:26 2026-02-20T21:12:31.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:12:31.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:12:31.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:12:31.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:12:31.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:28' 2026-02-20T21:12:31.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:12:31.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:12:31.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:12:31.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:12:31.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:12:31.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:12:31.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:12:31.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:12:31.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:12:31.353 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:12:31.353 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:12:31.353 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:12:31.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:26' 2026-02-20T21:12:31.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:12:26' 2026-02-20T21:12:31.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:31.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:31.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:31.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:12:31.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:12:26' 2026-02-20T21:12:31.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:31.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:31.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:12:31.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:12:26cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:12:28' 2026-02-20T21:12:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:12:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:12:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:26' 2026-02-20T21:12:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:12:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:12:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:12:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:12:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:12:31.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:31.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:31.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test 2026-02-20T21:12:31.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:31.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:31.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:31.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:12:31.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-20T21:12:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:12:32.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-20T21:12:32.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:32.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:32.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:32.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-20T21:12:32.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:32.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:32.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:32.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:12:32.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:12:32.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:32.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:12:33.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:12:33.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:33.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:33.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:33.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:12:33.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:12:33.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:12:33.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:12:33.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:12:33.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:12:33.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:33.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:12:33.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:12:33.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:12:33.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:33.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:12:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:12:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:12:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:12:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:12:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:12:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:12:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:12:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:12:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:12:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:12:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:12:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:12:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:12:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:12:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:12:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:12:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:12:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:12:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:12:33.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:12:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:12:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:12:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:12:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:12:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:12:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:12:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:12:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:12:33.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:33.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:33.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:33.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:12:34.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:12:34.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:12:34.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:12:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:12:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:12:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:12:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:12:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:12:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:12:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:12:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:12:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:12:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:12:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:12:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:12:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:12:34.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:12:34.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:12:34.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:12:34.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:12:34.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:12:34.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:12:34.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:12:34.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:12:34.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:34.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:12:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:12:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:12:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:12:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:12:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:12:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:12:34.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:34.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:34.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:34.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:12:34.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:12:34.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:12:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:12:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:12:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:12:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:34.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:12:36.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:12:36.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:36.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:36.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:12:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:12:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:12:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:12:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:12:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:12:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:12:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:12:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:12:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:12:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:12:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:12:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:12:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:12:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:12:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:12:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:12:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:12:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:12:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:12:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:12:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:12:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:12:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:12:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:12:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:12:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:12:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:12:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:12:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:12:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:12:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:12:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:12:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:12:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:12:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:12:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:12:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:12:36.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:36.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:36.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:36.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:12:36.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:12:36.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:12:36.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:12:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:12:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:12:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:12:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:12:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:12:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:12:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:12:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:12:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:12:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:12:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:12:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:12:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:12:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:12:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:12:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:12:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:12:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:12:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:12:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:12:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:12:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:12:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:12:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:12:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:12:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:12:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:12:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:12:36.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:36.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:36.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:36.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:12:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:12:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:12:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:12:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:12:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:12:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:12:40.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:12:40.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:40.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:40.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:40.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:12:40.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:12:40.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:12:40.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:12:40.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:12:40.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:40.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:12:40.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:12:40.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:12:40.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:40.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:12:40.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:12:40.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:12:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:12:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:12:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:12:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:12:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:12:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:12:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:12:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:12:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:12:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:12:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:12:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:12:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:12:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:12:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:12:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:12:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:12:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:12:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:12:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:12:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:12:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:12:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:12:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:12:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:12:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:12:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:12:40.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:40.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:40.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:40.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:12:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:12:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:12:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:12:40.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:40.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:12:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:12:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:12:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:12:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:12:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:12:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:12:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:12:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:12:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:12:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:12:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:12:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:12:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:12:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:12:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:12:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:12:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:12:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:12:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:12:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:12:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:12:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:12:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:12:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:12:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:12:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:12:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:12:40.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:40.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:40.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:40.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:12:40.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:12:40.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:12:40.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:12:40.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:12:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:12:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:12:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:12:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:12:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:12:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:12:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:12:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:12:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:12:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-20T21:12:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-20T21:12:42.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 87 2026-02-20T21:12:42.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:12:42.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:12:42.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:42.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:12:42.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:12:42.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:42.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:12:42.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:12:42.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:12:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:12:42.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:12:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:12:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:12:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:12:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:12:42.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:12:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=87 2026-02-20T21:12:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:12:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:12:43.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:12:43.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:43.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:43.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:43.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:12:43.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:12:43.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:12:43.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:12:43.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=87 2026-02-20T21:12:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 87 = 87 2026-02-20T21:12:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:12:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:12:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:12:44.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:12:44.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:44.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:12:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:12:44.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:12:44.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:12:44.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:12:44.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:12:44.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:12:44.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:43 2026-02-20T21:12:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:12:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:12:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:12:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621951,"replay_state":"idle"} 2026-02-20T21:12:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:36 2026-02-20T21:12:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:12:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 86 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.9697fba8-1f07-47ff-9db6-307300edd3da (peer_uuids:[]) 2026-02-20T21:12:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 87 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.7ecbcc90-4382-42ff-8d66-51d7b460998b (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:12:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:12:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:12:44.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:12:44.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:12:44.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:12:44.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:12:44.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:12:44.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:12:44.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:12:44.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:12:44.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:12:44.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:12:44.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:43' 2026-02-20T21:12:44.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:12:43' 2026-02-20T21:12:44.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:44.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:44.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:44.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:12:44.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:12:43' 2026-02-20T21:12:44.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:44.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:44.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:12:44.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary44044409mirror.0trial1992026-02-20 21:12:43cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621951,"replay_state":"idle"}2026-02-20 21:12:36' 2026-02-20T21:12:44.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:12:44.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:12:44.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:43' 2026-02-20T21:12:44.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:12:44.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:12:44.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:12:44.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:12:44.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:12:44.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:44.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:44.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-20T21:12:44.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:44.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:44.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:44.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:12:44.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:44.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:44.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:44.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:12:45.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-20T21:12:45.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:45.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:45.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:45.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:12:45.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:45.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:45.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:45.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:45.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:12:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:12:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:12:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:12:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621951,"replay_state":"idle"} 2026-02-20T21:12:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:12:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:36 2026-02-20T21:12:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:12:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:12:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:12:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:12:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:43' 2026-02-20T21:12:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:12:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:12:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:12:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621951,"replay_state":"idle"} 2026-02-20T21:12:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:12:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:12:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:12:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:12:45.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:12:45.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:12:45.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:12:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:12:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:36' 2026-02-20T21:12:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-20 21:12:36' 2026-02-20T21:12:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:12:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:12:36' 2026-02-20T21:12:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:45.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:12:45.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621951,"replay_state":"idle"}45294534mirror.1trial1992026-02-20 21:12:36cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:12:43' 2026-02-20T21:12:45.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:12:45.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:12:45.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:36' 2026-02-20T21:12:45.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:12:45.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:12:45.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:12:45.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:12:45.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:12:45.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:45.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:45.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster1 mirror test 2026-02-20T21:12:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-20T21:12:46.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:12:46.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-20T21:12:46.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:46.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:46.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:46.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-20T21:12:46.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:46.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:46.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:46.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:12:46.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:12:46.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:46.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:12:47.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:12:47.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:47.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:47.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:47.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:12:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:12:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:12:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:12:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:12:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:12:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:12:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:12:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:12:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:47.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:12:47.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:12:47.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:12:47.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:47.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:12:47.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:12:47.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:47.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:12:47.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:12:47.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:12:47.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:12:47.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:12:47.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:12:47.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:12:47.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:12:47.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:12:47.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:47.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:12:47.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:47.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:12:47.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:12:47.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:12:47.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:12:47.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:12:47.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:12:47.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:12:47.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:12:47.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:47.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:47.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:12:47.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:12:47.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:12:47.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:12:47.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:12:47.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:12:47.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:47.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:47.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:12:47.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:12:47.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:12:47.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:12:47.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:12:47.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:47.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:12:47.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:12:47.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:47.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:12:47.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:12:47.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:12:47.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:12:47.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:12:47.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:12:47.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:12:47.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:12:47.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:12:47.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:47.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:12:47.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:12:47.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:12:47.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:12:47.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:12:47.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:12:47.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:12:47.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:12:47.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:12:47.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:12:47.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:12:47.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:12:47.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:12:47.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:12:47.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:12:47.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:12:47.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:12:47.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:12:47.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:47.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:47.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:12:47.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:12:48.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:12:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:12:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:12:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:12:48.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:12:48.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:48.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:12:48.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:48.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:48.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:48.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:12:48.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:48.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:48.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:48.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:12:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:12:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:12:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:12:49.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:12:49.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:12:49.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:12:49.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:12:49.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:12:49.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:48 2026-02-20T21:12:49.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:12:49.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:12:49.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:12:49.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:12:49.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:47' 2026-02-20T21:12:49.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:12:49.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:12:49.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:12:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:12:49.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:12:49.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:12:49.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:12:49.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:12:49.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:12:49.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:12:49.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:12:49.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:12:49.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:48' 2026-02-20T21:12:49.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:12:48' 2026-02-20T21:12:49.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:12:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:12:48' 2026-02-20T21:12:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:12:49.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:12:48cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image demoted2026-02-20 21:12:47' 2026-02-20T21:12:49.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:12:49.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:12:49.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:48' 2026-02-20T21:12:49.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:12:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:12:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:12:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:12:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:12:49.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:12:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:12:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:12:50.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:12:50.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:50.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:50.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:50.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:12:50.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:50.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:50.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:50.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:50.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:12:50.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:12:50.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:12:50.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:12:50.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:12:50.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:12:50.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:47 2026-02-20T21:12:50.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:12:50.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:12:50.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:12:50.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:12:50.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:48' 2026-02-20T21:12:50.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:12:50.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:12:50.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:12:50.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image demoted 2026-02-20T21:12:50.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:12:50.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:12:50.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:12:50.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:12:50.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:12:50.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:12:50.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:12:50.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:12:50.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:47' 2026-02-20T21:12:50.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:12:47' 2026-02-20T21:12:50.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:50.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:50.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:50.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:12:50.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:12:47' 2026-02-20T21:12:50.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:50.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:50.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:12:50.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image demoted45294534mirror.1trial1992026-02-20 21:12:47cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:12:48' 2026-02-20T21:12:50.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:12:50.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:12:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:47' 2026-02-20T21:12:50.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:12:50.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:12:50.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:12:50.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:12:50.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:12:50.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:50.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:50.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster2 mirror test 2026-02-20T21:12:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:12:50.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-20T21:12:51.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:12:51.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:12:51.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:12:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:12:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:12:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:12:52.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:12:52.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:52.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:52.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:52.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:12:52.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:12:52.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:12:52.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:12:52.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:12:52.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:52.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:12:52.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:12:52.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:12:52.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:12:52.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:52.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:12:52.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:52.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:12:52.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:52.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:12:52.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:12:52.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:52.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:12:52.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:52.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:12:52.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:12:52.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:12:52.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:12:52.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:12:52.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:52.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:12:52.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:12:52.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:12:52.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:52.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:52.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:12:52.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:12:52.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:12:52.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:52.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:12:52.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:12:52.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:12:52.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:12:52.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:52.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:52.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:12:52.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:12:52.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:12:52.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:12:52.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:52.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:52.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:52.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:52.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:12:52.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:12:52.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:12:52.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:12:52.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:12:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:12:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:12:54.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:12:54.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:12:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:12:54.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:12:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:12:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:12:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:12:54.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:54.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:12:54.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:12:54.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:12:54.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:54.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:12:54.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:54.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:12:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:12:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:12:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:12:54.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:12:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:12:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:12:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:12:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:12:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:12:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:12:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:12:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:12:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:12:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:12:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:12:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:12:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:12:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:12:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:12:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:12:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:54.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:12:54.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:12:54.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:12:54.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:12:54.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:12:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:12:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:12:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:12:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:12:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:12:55.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:12:55.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:12:55.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:55.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:12:55.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:12:55.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:55.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:12:55.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:12:55.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:55.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:12:55.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:12:55.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:12:55.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:55.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:55.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:55.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:12:57.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 90 2026-02-20T21:12:57.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:12:57.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:12:57.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:57.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:12:57.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:12:57.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:57.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:12:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:12:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:12:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:12:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:12:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:12:57.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:57.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:57.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:57.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:12:57.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:12:57.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:12:57.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:12:57.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=90 2026-02-20T21:12:57.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:12:57.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:57.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:12:58.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:12:58.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:58.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:58.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:58.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:12:58.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:12:58.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:12:58.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:12:58.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=90 2026-02-20T21:12:58.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 90 = 90 2026-02-20T21:12:58.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:58.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-20T21:12:58.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:58.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:58.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:58.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:12:58.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:58.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:58.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:58.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:12:59.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-20T21:12:59.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:12:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:12:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:12:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:12:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:12:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:12:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:12:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:56 2026-02-20T21:12:59.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:12:59.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:12:59.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:12:59.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621975,"remote_snapshot_timestamp":1771621975,"replay_state":"idle"} 2026-02-20T21:12:59.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:58 2026-02-20T21:12:59.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:12:59.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 89 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.4fa8cb5f-b5da-43e9-8fcd-f791bcdce7b5 (peer_uuids:[]) 2026-02-20T21:12:59.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 90 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.af5a432c-b95c-49ea-9940-0f957a55b5e0 (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:12:59.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:12:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:12:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:12:59.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:12:59.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:12:59.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:12:59.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:12:59.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:12:59.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:12:59.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:12:59.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:12:59.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:12:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:56' 2026-02-20T21:12:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-20 21:12:56' 2026-02-20T21:12:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:12:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:12:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:12:56' 2026-02-20T21:12:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:12:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:12:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:12:59.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:12:56cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621975,"remote_snapshot_timestamp":1771621975,"replay_state":"idle"}2026-02-20 21:12:58' 2026-02-20T21:12:59.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:12:59.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:12:59.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:56' 2026-02-20T21:12:59.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:12:59.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:12:59.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:12:59.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:12:59.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:12:59.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:59.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:12:59.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:12:59.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:12:59.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:12:59.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:12:59.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:12:59.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:12:59.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:12:59.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:59.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:13:00.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:13:00.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:00.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:00.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:00.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:13:00.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:00.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:00.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:00.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:00.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:13:00.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:13:00.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:13:00.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:13:00.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621975,"remote_snapshot_timestamp":1771621975,"replay_state":"idle"} 2026-02-20T21:13:00.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:13:00.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:58 2026-02-20T21:13:00.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:13:00.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:13:00.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:13:00.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:13:00.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:12:56' 2026-02-20T21:13:00.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:13:00.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:13:00.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:13:00.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621975,"remote_snapshot_timestamp":1771621975,"replay_state":"idle"} 2026-02-20T21:13:00.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:13:00.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:13:00.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:13:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:13:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:13:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:13:00.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:13:00.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:13:00.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:58' 2026-02-20T21:13:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:12:58' 2026-02-20T21:13:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:13:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:12:58' 2026-02-20T21:13:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:13:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771621975,"remote_snapshot_timestamp":1771621975,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:12:58cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:12:56' 2026-02-20T21:13:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:13:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:13:00.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:12:58' 2026-02-20T21:13:00.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:13:00.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:13:00.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:13:00.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:13:00.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:13:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:13:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test 2026-02-20T21:13:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-20T21:13:01.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:13:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:13:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:13:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:13:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:13:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:13:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:13:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:13:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:13:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:13:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:13:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:13:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:13:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:13:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:13:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:13:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:13:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:13:02.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:13:02.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:02.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:13:02.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:13:02.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:02.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:13:02.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:13:02.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:13:02.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:13:02.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:13:02.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:13:02.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:13:02.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:02.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:13:02.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:13:02.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:13:02.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:02.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:13:02.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:13:02.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:13:02.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:13:02.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:13:02.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:13:02.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:13:02.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:13:02.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:13:02.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:02.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:02.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:13:02.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:13:02.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:13:02.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:13:02.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:13:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:13:02.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:13:02.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:13:02.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:13:02.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:13:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:13:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:13:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:02.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:13:03.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:13:03.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:03.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:03.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:03.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:03.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:13:04.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:13:04.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:13:04.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:13:04.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:13:04.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:13:04.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:03 2026-02-20T21:13:04.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:13:04.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:13:04.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:13:04.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:13:04.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:01' 2026-02-20T21:13:04.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:13:04.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:13:04.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:13:04.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:13:04.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:13:04.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:13:04.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:13:04.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:13:04.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:13:04.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:13:04.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:13:04.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:13:04.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:03' 2026-02-20T21:13:04.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:13:03' 2026-02-20T21:13:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:13:03' 2026-02-20T21:13:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:13:04.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:13:03cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:13:01' 2026-02-20T21:13:04.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:13:04.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:13:04.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:03' 2026-02-20T21:13:04.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:13:04.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:13:04.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:13:04.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:13:04.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:13:04.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:04.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:04.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:13:04.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:04.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:04.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:04.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:04.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:04.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:04.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:04.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:13:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:13:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:05.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:05.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:05.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:13:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:13:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:13:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:13:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:13:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:13:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:01 2026-02-20T21:13:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:13:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:13:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:13:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:13:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:03' 2026-02-20T21:13:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:13:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:13:05.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:13:05.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:13:05.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:13:05.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:13:05.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:13:05.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:13:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:13:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:13:05.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:13:05.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:13:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:01' 2026-02-20T21:13:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:13:01' 2026-02-20T21:13:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:13:01' 2026-02-20T21:13:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:05.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:05.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:13:05.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:13:01cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:13:03' 2026-02-20T21:13:05.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:13:05.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:13:05.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:01' 2026-02-20T21:13:05.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:13:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:13:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:13:05.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:13:05.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:13:05.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:05.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:05.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test 2026-02-20T21:13:05.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:05.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:05.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:05.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:13:05.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-20T21:13:06.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:13:06.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-20T21:13:06.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-20T21:13:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:13:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:13:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:13:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:13:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:13:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:13:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:13:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:13:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:13:07.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:07.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:13:07.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:13:07.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:13:07.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:13:07.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:07.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:13:07.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:13:07.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:13:07.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:07.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:13:07.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:13:07.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:07.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:13:07.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:13:07.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:13:07.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:13:07.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:13:07.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:13:07.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:13:07.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:13:07.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:13:07.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:07.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:13:07.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:07.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:13:07.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:13:07.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:13:07.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:13:07.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:13:07.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:13:07.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:13:07.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:13:07.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:07.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:07.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:13:07.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:13:07.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:13:07.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:13:07.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:13:07.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:13:07.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:07.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:07.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:07.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:13:07.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:13:07.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:13:07.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:13:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:13:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:13:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:13:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:13:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:13:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:13:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:13:07.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:13:07.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:13:07.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:13:07.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:13:07.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:07.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:13:07.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:13:07.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:13:07.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:13:07.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:13:07.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:13:07.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:13:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:13:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:13:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:13:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:13:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:13:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:13:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:13:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:13:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:13:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:13:07.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:07.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:07.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:07.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:13:08.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:13:08.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:13:08.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:13:08.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:13:08.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:13:08.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:13:08.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:13:08.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:08.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:13:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:13:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:13:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:13:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:13:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:13:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-20T21:13:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-20T21:13:11.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 93 2026-02-20T21:13:11.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:13:11.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:13:11.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:11.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:13:11.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:13:11.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:11.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:13:11.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:13:11.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:11.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:13:11.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:11.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:13:11.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:11.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:13:11.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:13:11.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:11.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:11.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:11.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:13:11.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:13:11.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:13:11.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:13:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=93 2026-02-20T21:13:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:13:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:13:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:13:12.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:13:12.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:12.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:12.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:12.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:13:12.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:13:12.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:13:12.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:13:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=93 2026-02-20T21:13:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 93 = 93 2026-02-20T21:13:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:13:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:13:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:13:13.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:13:13.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:13.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:13.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:13.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:13:13.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:13.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:13.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:13.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:13.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:13:13.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:13:13.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:13:13.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:13:13.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:13:13.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:13:13.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:13 2026-02-20T21:13:13.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:13:13.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:13:13.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:13:13.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621985,"replay_state":"idle"} 2026-02-20T21:13:13.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:06 2026-02-20T21:13:13.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:13:13.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 93 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.2b37c160-de8d-4fc3-abb5-2a1d19074af2 (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:13:13.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:13:13.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:13:13.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:13:13.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:13:13.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:13:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:13:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:13:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:13:13.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:13:13.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:13:13.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:13:13.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:13:13.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:13' 2026-02-20T21:13:13.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:13:13' 2026-02-20T21:13:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:13:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:13:13' 2026-02-20T21:13:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:13:13.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary44044409mirror.0trial1992026-02-20 21:13:13cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621985,"replay_state":"idle"}2026-02-20 21:13:06' 2026-02-20T21:13:13.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:13:13.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:13:13.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:13' 2026-02-20T21:13:13.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:13:13.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:13:13.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:13:13.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:13:13.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:13:13.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:13.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:13.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-20T21:13:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:13:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:13:14.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-20T21:13:14.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:14.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:14.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:14.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:13:14.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:14.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:14.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:14.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:14.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:13:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:13:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:13:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:13:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621985,"replay_state":"idle"} 2026-02-20T21:13:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:13:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:06 2026-02-20T21:13:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:13:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:13:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:13:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:13:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:13' 2026-02-20T21:13:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:13:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:13:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:13:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621985,"replay_state":"idle"} 2026-02-20T21:13:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:13:14.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:13:14.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:13:14.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:13:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:13:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:13:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:13:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:13:14.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:06' 2026-02-20T21:13:14.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-20 21:13:06' 2026-02-20T21:13:14.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:14.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:14.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:14.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:13:14.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:13:06' 2026-02-20T21:13:14.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:14.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:14.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:13:14.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771621985,"replay_state":"idle"}45294534mirror.1trial1992026-02-20 21:13:06cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:13:13' 2026-02-20T21:13:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:13:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:13:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:06' 2026-02-20T21:13:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:13:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:13:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:13:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:13:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:13:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster1 mirror test 2026-02-20T21:13:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:14.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:14.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:14.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-20T21:13:14.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:13:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-20T21:13:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-20T21:13:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:13:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:13:14.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:14.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:13:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:13:15.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:15.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:15.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:15.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:13:15.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:13:15.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:13:15.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:13:15.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:13:15.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:13:15.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:15.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:13:15.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:13:15.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:13:15.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:15.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:13:15.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:13:15.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:13:15.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:15.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:13:15.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:13:15.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:15.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:13:15.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:13:15.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:13:15.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:13:15.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:13:15.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:13:15.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:13:15.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:13:15.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:13:15.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:15.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:13:15.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:15.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:13:15.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:13:15.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:13:15.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:13:15.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:13:15.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:13:15.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:13:15.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:13:15.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:15.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:15.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:13:15.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:13:15.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:13:15.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:13:15.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:13:15.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:13:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:13:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:13:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:13:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:13:16.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:13:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:13:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:13:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:13:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:13:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:13:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:13:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:13:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:13:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:13:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:13:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:13:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:13:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:13:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:13:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:13:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:13:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:13:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:13:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:13:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:13:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:16.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:13:16.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:13:16.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:13:16.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:13:16.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:13:16.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:13:16.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:13:16.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:16.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:16.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:16.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:13:16.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:13:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:13:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:13:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:13:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:13:16.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:13:16.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:13:16.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:16.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:13:18.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:13:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:13:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:13:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:13:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:13:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:13:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:13:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:13:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:13:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:13:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:13:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:13:18.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:13:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:13:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:13:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:13:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:13:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:13:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:13:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:13:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:13:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:13:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:13:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:13:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:13:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:13:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:13:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:13:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:13:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:13:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:13:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:13:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:13:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:13:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:13:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:13:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:13:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:13:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:13:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:13:18.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:13:18.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:13:18.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:13:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:13:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:13:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:13:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:13:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:13:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:13:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:13:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:13:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:13:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:13:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:13:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:13:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:13:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:13:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:13:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:13:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:13:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:13:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:13:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:13:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:13:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:13:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:13:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:13:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:13:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:13:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:13:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:13:18.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:18.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:18.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:18.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:13:18.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:13:18.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:13:18.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:13:18.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:13:18.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:13:18.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:18.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:13:18.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:18.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:18.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:18.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:18.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:18.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:18.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:18.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:13:19.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:13:19.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:19.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:19.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:19.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:19.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:19.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:19.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:19.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:19.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:13:19.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:13:19.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:13:19.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:13:19.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:13:19.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:13:19.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:18 2026-02-20T21:13:19.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:13:19.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:13:19.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:13:19.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:13:19.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:16' 2026-02-20T21:13:19.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:13:19.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:13:19.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:13:19.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:13:19.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:13:19.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:13:19.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:13:19.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:13:19.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:13:19.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:13:19.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:13:19.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:13:19.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:18' 2026-02-20T21:13:19.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:13:18' 2026-02-20T21:13:19.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:19.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:19.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:19.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:19.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:13:18' 2026-02-20T21:13:19.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:19.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:19.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:13:19.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:13:18cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:13:16' 2026-02-20T21:13:19.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:13:19.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:13:19.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:18' 2026-02-20T21:13:19.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:13:19.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:13:19.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:13:19.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:13:19.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:13:19.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:19.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:19.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:13:19.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:19.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:19.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:19.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:19.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:19.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:19.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:19.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:13:20.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:13:20.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:20.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:20.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:20.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:20.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:20.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:20.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:20.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:20.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:13:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:13:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:13:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:13:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:13:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:13:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:16 2026-02-20T21:13:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:13:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:13:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:13:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:13:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:18' 2026-02-20T21:13:20.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:13:20.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:13:20.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:13:20.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:13:20.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:13:20.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:13:20.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:13:20.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:13:20.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:13:20.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:13:20.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:13:20.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:13:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:16' 2026-02-20T21:13:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:13:16' 2026-02-20T21:13:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:13:16' 2026-02-20T21:13:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:13:20.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:13:16cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:13:18' 2026-02-20T21:13:20.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:13:20.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:13:20.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:16' 2026-02-20T21:13:20.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:13:20.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:13:20.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:13:20.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:13:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:13:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster2 mirror test 2026-02-20T21:13:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:20.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:20.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:20.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:13:20.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-20T21:13:21.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:13:21.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:13:21.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:21.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:13:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:13:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:13:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:13:22.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:13:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:13:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:13:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:13:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:13:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:13:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:13:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:13:22.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:13:22.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:13:22.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:22.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:13:22.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:13:22.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:13:22.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:22.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:13:22.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:13:22.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:22.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:13:22.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:13:22.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:13:22.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:13:22.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:13:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:13:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:13:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:13:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:13:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:13:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:13:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:13:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:13:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:13:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:13:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:13:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:13:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:13:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:13:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:13:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:13:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:13:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:13:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:13:22.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:22.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:22.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:22.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:13:22.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:13:22.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:13:22.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:13:22.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:13:22.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:13:22.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:22.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:13:24.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:13:24.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:24.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:24.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:24.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:13:24.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:13:24.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:13:24.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:13:24.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:13:24.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:13:24.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:24.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:13:24.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:13:24.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:13:24.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:24.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:13:24.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:13:24.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:13:24.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:24.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:13:24.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:13:24.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:24.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:13:24.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:13:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:13:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:13:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:13:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:13:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:13:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:13:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:13:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:13:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:13:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:13:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:13:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:13:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:13:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:13:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:13:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:13:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:13:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:13:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:13:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:13:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:13:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:13:24.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:24.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:24.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:24.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:13:25.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:13:25.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:13:25.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:13:25.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:13:25.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:13:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:13:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:13:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:13:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:13:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:13:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:13:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:13:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:13:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:13:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:25.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:13:28.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 96 2026-02-20T21:13:28.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:13:28.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:13:28.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:28.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:13:28.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:13:28.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:28.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:28.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:13:28.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:28.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:13:28.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:13:28.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:13:28.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:28.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:13:28.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:13:28.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:28.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:28.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:28.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:13:28.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:13:28.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:13:28.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:13:28.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=96 2026-02-20T21:13:28.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:13:28.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:13:28.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:13:28.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:13:28.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:28.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:28.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:28.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:13:28.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:13:28.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:13:28.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:13:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=96 2026-02-20T21:13:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 96 = 96 2026-02-20T21:13:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:28.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-20T21:13:28.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:28.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:28.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:28.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:13:28.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:28.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:28.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:28.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:13:29.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-20T21:13:29.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:29.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:29.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:29.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:13:29.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:29.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:29.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:29.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:29.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:13:29.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:13:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:13:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:13:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:13:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:13:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:26 2026-02-20T21:13:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:13:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:13:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:13:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622005,"remote_snapshot_timestamp":1771622005,"replay_state":"idle"} 2026-02-20T21:13:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:28 2026-02-20T21:13:29.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:13:29.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 96 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.008744aa-afc0-4b7d-bb9c-ea6b2ff64468 (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:13:29.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:13:29.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:13:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:13:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:13:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:13:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:13:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:13:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:13:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:13:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:13:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:13:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:13:29.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:26' 2026-02-20T21:13:29.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-20 21:13:26' 2026-02-20T21:13:29.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:29.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:29.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:29.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:13:29.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:13:26' 2026-02-20T21:13:29.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:29.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:29.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:13:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:13:26cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622005,"remote_snapshot_timestamp":1771622005,"replay_state":"idle"}2026-02-20 21:13:28' 2026-02-20T21:13:29.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:13:29.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:13:29.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:26' 2026-02-20T21:13:29.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:13:29.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:13:29.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:13:29.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:13:29.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:13:29.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:29.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:29.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:13:29.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:29.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:29.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:29.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:13:29.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:29.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:29.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:29.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:13:30.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:13:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:13:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:13:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:13:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:13:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:13:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622005,"remote_snapshot_timestamp":1771622005,"replay_state":"idle"} 2026-02-20T21:13:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:13:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:28 2026-02-20T21:13:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:13:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:13:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:13:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:13:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:26' 2026-02-20T21:13:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:13:30.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:13:30.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:13:30.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622005,"remote_snapshot_timestamp":1771622005,"replay_state":"idle"} 2026-02-20T21:13:30.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:13:30.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:13:30.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:13:30.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:13:30.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:13:30.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:13:30.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:13:30.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:13:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:28' 2026-02-20T21:13:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:13:28' 2026-02-20T21:13:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:13:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:13:28' 2026-02-20T21:13:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:13:30.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622005,"remote_snapshot_timestamp":1771622005,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:13:28cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:13:26' 2026-02-20T21:13:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:13:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:13:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:28' 2026-02-20T21:13:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:13:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:13:30.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:13:30.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:13:30.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:13:30.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:13:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test 2026-02-20T21:13:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-20T21:13:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:13:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:13:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:13:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:13:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:13:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:13:32.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:13:32.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:32.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:32.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:13:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:13:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:13:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:13:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:13:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:13:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:13:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:13:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:13:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:13:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:13:32.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:13:32.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:32.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:13:32.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:13:32.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:32.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:13:32.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:13:32.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:13:32.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:13:32.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:13:32.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:13:32.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:13:32.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:32.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:13:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:13:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:13:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:13:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:13:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:13:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:13:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:13:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:13:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:13:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:13:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:13:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:13:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:13:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:13:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:13:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:13:32.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:32.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:32.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:32.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:13:32.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:13:32.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:13:32.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:13:32.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:13:32.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:13:32.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:13:32.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:13:32.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:32.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:13:34.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:13:34.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:34.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:34.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:34.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:13:34.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:13:34.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:13:34.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:13:34.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:13:34.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:34.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:13:34.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:13:34.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:13:34.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:34.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:13:34.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:13:34.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:13:34.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:13:34.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:34.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:13:34.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:13:34.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:34.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:13:34.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:13:34.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:13:34.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:13:34.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:13:34.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:13:34.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:13:34.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:34.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:13:34.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:13:34.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:13:34.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:34.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:13:34.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:13:34.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:13:34.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:13:34.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:13:34.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:13:34.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:13:34.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:13:34.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:13:34.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:34.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:34.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:13:34.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:13:34.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:13:34.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:13:34.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:13:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:13:34.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:13:34.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:13:34.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:13:34.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:13:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:13:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:13:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:34.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:34.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:13:35.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:13:35.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:35.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:35.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:35.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:35.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:35.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:35.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:35.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:35.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:13:35.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:13:35.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:13:35.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:13:35.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:13:35.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:13:35.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:33 2026-02-20T21:13:35.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:13:35.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:13:35.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:13:35.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:13:35.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:31' 2026-02-20T21:13:35.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:13:35.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:13:35.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:13:35.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:13:35.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:13:35.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:13:35.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:13:35.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:13:35.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:13:35.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:13:35.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:13:35.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:13:35.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:33' 2026-02-20T21:13:35.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:13:33' 2026-02-20T21:13:35.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:35.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:35.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:35.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:35.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:13:33' 2026-02-20T21:13:35.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:35.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:35.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:13:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:13:33cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:13:31' 2026-02-20T21:13:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:13:35.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:13:35.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:33' 2026-02-20T21:13:35.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:13:35.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:13:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:13:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:13:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:13:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:13:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:13:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:13:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:13:36.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:13:36.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:13:36.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:13:36.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:13:36.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:13:36.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:31 2026-02-20T21:13:36.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:13:36.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:13:36.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:13:36.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:13:36.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:33' 2026-02-20T21:13:36.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:13:36.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:13:36.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:13:36.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:13:36.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:13:36.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:13:36.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:13:36.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:13:36.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:13:36.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:13:36.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:13:36.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:13:36.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:31' 2026-02-20T21:13:36.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:13:31' 2026-02-20T21:13:36.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:36.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:36.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:36.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:36.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:13:31' 2026-02-20T21:13:36.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:36.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:36.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:13:36.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:13:31cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:13:33' 2026-02-20T21:13:36.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:13:36.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:13:36.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:31' 2026-02-20T21:13:36.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:13:36.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:13:36.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:13:36.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:13:36.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:13:36.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:36.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:36.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test 2026-02-20T21:13:36.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:36.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:36.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:36.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:13:36.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-20T21:13:37.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:13:37.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-20T21:13:37.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:37.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:37.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:37.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-20T21:13:37.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:37.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:37.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:37.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:13:37.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:13:37.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:37.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:13:38.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:13:38.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:38.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:38.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:38.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:13:38.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:13:38.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:13:38.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:13:38.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:13:38.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:13:38.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:38.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:13:38.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:13:38.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:13:38.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:38.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:13:38.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:13:38.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:13:38.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:38.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:13:38.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:13:38.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:38.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:13:38.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:13:38.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:13:38.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:13:38.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:13:38.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:13:38.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:13:38.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:13:38.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:13:38.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:38.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:13:38.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:38.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:13:38.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:13:38.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:13:38.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:13:38.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:13:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:13:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:13:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:13:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:13:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:13:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:13:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:13:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:13:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:13:38.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:38.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:38.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:38.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:13:38.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:13:38.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:13:38.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:13:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:38.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:13:38.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:13:38.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:38.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:13:38.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:13:38.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:13:38.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:13:38.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:13:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:13:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:13:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:13:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:13:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:13:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:13:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:13:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:13:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:13:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:13:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:13:38.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:13:38.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:13:38.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:13:38.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:38.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:13:38.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:13:38.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:13:38.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:13:38.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:13:38.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:13:38.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:13:38.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:38.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:38.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:38.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:13:39.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:13:39.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:13:39.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:13:39.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:13:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:13:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:13:41.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:13:41.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:41.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:41.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:41.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:13:41.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:13:41.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:13:41.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:13:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:13:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:13:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:13:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:13:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:13:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:13:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:13:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:13:41.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:41.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:13:41.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:13:41.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:41.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:13:41.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:13:41.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:13:41.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:13:41.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:13:41.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:13:41.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:13:41.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:13:41.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:13:41.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:41.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:13:41.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:41.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:13:41.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:13:41.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:13:41.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:13:41.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:13:41.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:13:41.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:13:41.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:13:41.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:41.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:41.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:13:41.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:13:41.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:13:41.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:13:41.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:13:41.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:13:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:41.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:13:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:13:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:13:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:13:41.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:41.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:13:41.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:13:41.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:41.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:13:41.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:13:41.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:13:41.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:13:41.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:13:41.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:13:41.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:13:41.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:13:41.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:13:41.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:41.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:13:41.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:13:41.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:13:41.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:13:41.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:13:41.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:13:41.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:13:41.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:13:41.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:13:41.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:13:41.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:41.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:13:41.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:13:41.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:13:41.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:13:41.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:13:41.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:13:41.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:13:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:13:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:13:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:13:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:13:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:13:41.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:13:41.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:41.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:13:45.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:13:45.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:45.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:45.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:45.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:13:45.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:13:45.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:13:45.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:13:45.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:13:45.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:45.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:13:45.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:13:45.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:13:45.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:45.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:13:45.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:13:45.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:13:45.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:13:45.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:45.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:13:45.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:13:45.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:45.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:13:45.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:13:45.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:13:45.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:13:45.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:13:45.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:13:45.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:13:45.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:13:45.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:13:45.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:45.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:13:45.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:45.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:13:45.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:13:45.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:13:45.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:13:45.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:13:45.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:13:45.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:13:45.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:13:45.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:45.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:45.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:13:45.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:13:45.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:13:45.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:13:45.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:13:45.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:13:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:13:45.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:13:45.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:13:45.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:13:45.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:45.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:13:45.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:13:45.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:45.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:13:45.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:13:45.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:13:45.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:13:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:13:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:13:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:13:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:13:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:13:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:13:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:13:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:13:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:13:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:13:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:13:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:13:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:13:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:13:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:13:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:13:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:13:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:13:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:13:45.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:13:45.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:13:45.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:13:45.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:45.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:45.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:45.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:13:45.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:13:45.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:13:45.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:13:45.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:13:45.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:13:45.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:13:45.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:13:45.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:45.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:13:45.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:13:45.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:45.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:13:45.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:13:45.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:45.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:13:45.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:13:45.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-20T21:13:45.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:45.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:45.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:45.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-20T21:13:48.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 99 2026-02-20T21:13:48.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:13:48.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:13:48.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:48.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:13:48.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:13:48.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:48.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:13:48.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:13:48.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:48.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:13:48.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:48.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:13:48.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:48.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:13:48.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:13:48.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:48.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:48.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:48.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:13:48.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:13:48.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:13:48.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:13:48.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=99 2026-02-20T21:13:48.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:13:48.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:13:48.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:13:49.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:13:49.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:49.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:49.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:49.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:13:49.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:13:49.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:13:49.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:13:49.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=99 2026-02-20T21:13:49.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 99 = 99 2026-02-20T21:13:49.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:49.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:13:49.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:49.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:49.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:49.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:13:49.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:49.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:49.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:49.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:13:50.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:13:50.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:50.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:50.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:50.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:13:50.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:50.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:50.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:50.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:50.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:13:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:13:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:13:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:13:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:13:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:13:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:48 2026-02-20T21:13:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:13:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:13:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:13:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622016,"replay_state":"idle"} 2026-02-20T21:13:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:41 2026-02-20T21:13:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:13:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 99 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.234ccae8-7785-435a-8ab6-7e1b03f14837 (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:13:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:13:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:13:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:13:50.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:13:50.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:13:50.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:13:50.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:13:50.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:13:50.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:13:50.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:13:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:13:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:13:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:48' 2026-02-20T21:13:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:13:48' 2026-02-20T21:13:50.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:50.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:50.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:50.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:13:50.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:13:48' 2026-02-20T21:13:50.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:50.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:50.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:13:50.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary44044409mirror.0trial1992026-02-20 21:13:48cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622016,"replay_state":"idle"}2026-02-20 21:13:41' 2026-02-20T21:13:50.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:13:50.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:13:50.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:48' 2026-02-20T21:13:50.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:13:50.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:13:50.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:13:50.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:13:50.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:13:50.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:50.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:50.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-20T21:13:50.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:50.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:50.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:50.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:13:50.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:50.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:50.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:50.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:13:51.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-20T21:13:51.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:51.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:51.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:51.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:13:51.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:51.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:51.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:51.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:51.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:13:51.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:13:51.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:13:51.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:13:51.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622016,"replay_state":"idle"} 2026-02-20T21:13:51.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:13:51.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:41 2026-02-20T21:13:51.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:13:51.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:13:51.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:13:51.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:13:51.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:48' 2026-02-20T21:13:51.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:13:51.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:13:51.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:13:51.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622016,"replay_state":"idle"} 2026-02-20T21:13:51.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:13:51.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:13:51.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:13:51.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:13:51.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:13:51.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:13:51.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:13:51.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:13:51.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:41' 2026-02-20T21:13:51.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-20 21:13:41' 2026-02-20T21:13:51.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:51.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:51.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:51.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:13:51.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:13:41' 2026-02-20T21:13:51.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:51.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:51.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:13:51.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622016,"replay_state":"idle"}45294534mirror.1trial1992026-02-20 21:13:41cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:13:48' 2026-02-20T21:13:51.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:13:51.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:13:51.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:41' 2026-02-20T21:13:51.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:13:51.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:13:51.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:13:51.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:13:51.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:13:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster1 mirror test 2026-02-20T21:13:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:51.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:51.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:51.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-20T21:13:51.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:13:51.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-20T21:13:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-20T21:13:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:13:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:13:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:13:52.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:13:52.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:52.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:52.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:52.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:13:52.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:13:52.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:13:52.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:13:52.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:13:52.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:13:52.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:52.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:13:52.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:13:52.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:13:52.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:52.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:13:52.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:13:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:13:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:13:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:13:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:13:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:13:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:13:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:13:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:13:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:13:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:13:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:13:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:13:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:13:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:13:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:13:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:13:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:13:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:13:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:13:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:13:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:13:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:13:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:13:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:13:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:13:52.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:13:52.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:13:52.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:52.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:52.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:52.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:13:53.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:13:53.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:13:53.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:13:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:13:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:13:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:13:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:13:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:13:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:13:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:13:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:13:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:13:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:13:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:13:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:13:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:13:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:13:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:13:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:13:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:13:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:13:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:13:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:13:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:13:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:13:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:13:53.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:13:53.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:13:53.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:13:53.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:13:53.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:13:53.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:53.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:53.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:53.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:13:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:13:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:13:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:13:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:13:53.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:13:53.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:13:53.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:13:53.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:53.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:13:55.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:13:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:13:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:13:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:13:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:13:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:13:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:13:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:13:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:13:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:13:55.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:55.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:13:55.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:13:55.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:13:55.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:55.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:13:55.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:13:55.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:55.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:13:55.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:13:55.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:13:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:13:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:13:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:13:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:13:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:13:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:13:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:13:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:13:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:13:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:13:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:13:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:13:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:13:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:13:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:13:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:13:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:13:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:13:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:13:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:13:55.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:13:55.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:55.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:55.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:13:55.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:13:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:13:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:13:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:13:55.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:55.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:13:55.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:13:55.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:13:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:13:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:13:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:13:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:13:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:13:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:13:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:13:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:13:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:13:55.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:13:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:13:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:13:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:13:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:13:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:13:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:13:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:13:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:13:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:13:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:13:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:13:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:13:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:13:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:13:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:13:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:13:55.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:55.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:55.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:13:55.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:13:55.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:13:55.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:13:55.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:13:55.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:13:55.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:13:55.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:55.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:13:55.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:13:56.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:13:56.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:56.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:56.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:56.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:56.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:56.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:56.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:56.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:56.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:13:56.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:13:56.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:13:56.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:13:56.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:13:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:13:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:53 2026-02-20T21:13:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:13:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:13:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:13:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:13:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:53' 2026-02-20T21:13:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:13:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:13:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:13:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:13:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:13:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:13:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:13:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:13:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:13:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:13:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:13:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:13:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:53' 2026-02-20T21:13:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:13:53' 2026-02-20T21:13:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:13:53' 2026-02-20T21:13:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:13:56.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:13:53cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image demoted2026-02-20 21:13:53' 2026-02-20T21:13:56.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:13:56.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:13:56.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:53' 2026-02-20T21:13:56.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:13:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:13:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:13:56.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:13:56.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:13:56.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:56.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:56.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:13:56.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:56.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:56.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:56.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:56.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:56.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:56.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:56.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:13:57.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:13:57.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:57.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:57.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:57.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:57.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:13:57.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:13:57.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:57.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:57.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:13:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:13:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:13:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:13:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:13:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:13:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:56 2026-02-20T21:13:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:13:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:13:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:13:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:13:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:53' 2026-02-20T21:13:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:13:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:13:57.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:13:57.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:13:57.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:13:57.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:13:57.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:13:57.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:13:57.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:13:57.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:13:57.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:13:57.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:13:57.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:56' 2026-02-20T21:13:57.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:13:56' 2026-02-20T21:13:57.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:57.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:57.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:57.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:13:57.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:13:56' 2026-02-20T21:13:57.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:13:57.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:13:57.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:13:57.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:13:56cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:13:53' 2026-02-20T21:13:57.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:13:57.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:13:57.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:56' 2026-02-20T21:13:57.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:13:57.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:13:57.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:13:57.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:13:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:13:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster2 mirror test 2026-02-20T21:13:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:13:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-20T21:13:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:13:58.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:13:58.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:58.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:58.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:58.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:13:58.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:58.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:58.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:58.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:13:58.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:13:58.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:58.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:13:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:13:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:13:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:13:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:13:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:13:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:13:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:13:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:13:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:13:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:13:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:13:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:13:59.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:13:59.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:13:59.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:59.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:13:59.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:13:59.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:13:59.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:13:59.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:13:59.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:13:59.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:13:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:13:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:13:59.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:13:59.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:13:59.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:13:59.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:13:59.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:13:59.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:13:59.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:13:59.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:13:59.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:13:59.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:13:59.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:59.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:13:59.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:13:59.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:59.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:13:59.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:13:59.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:13:59.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:13:59.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:13:59.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:13:59.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:14:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 102 2026-02-20T21:14:02.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:14:02.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:14:02.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:02.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:14:02.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:14:02.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:02.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:14:02.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:14:02.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:14:02.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:14:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:14:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:14:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:14:02.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:14:02.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:14:02.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:02.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:02.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:02.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:14:02.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:14:02.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:14:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:14:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=102 2026-02-20T21:14:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:14:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:14:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:14:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:14:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:14:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:14:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:14:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:14:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=102 2026-02-20T21:14:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 102 = 102 2026-02-20T21:14:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-20T21:14:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:14:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:14:04.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-20T21:14:04.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:04.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:04.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:04.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:14:04.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:04.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:04.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:04.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:04.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:14:04.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:14:04.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:14:04.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:14:04.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:14:04.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:14:04.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:01 2026-02-20T21:14:04.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:14:04.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:14:04.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:14:04.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622037,"replay_state":"idle"} 2026-02-20T21:14:04.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:58 2026-02-20T21:14:04.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:14:04.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 102 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.6bd7b09c-f115-45ce-ab2d-2a231bfede4a (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:14:04.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:14:04.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:14:04.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:14:04.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:14:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:14:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:14:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:14:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:14:04.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:14:04.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:14:04.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:14:04.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:14:04.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:01' 2026-02-20T21:14:04.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-20 21:14:01' 2026-02-20T21:14:04.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:14:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:14:01' 2026-02-20T21:14:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:14:04.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:14:01cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622037,"replay_state":"idle"}2026-02-20 21:13:58' 2026-02-20T21:14:04.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:14:04.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:14:04.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:01' 2026-02-20T21:14:04.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:14:04.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:14:04.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:14:04.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:14:04.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:14:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:14:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:14:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:14:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:14:05.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:14:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:14:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:14:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:14:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:14:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622037,"replay_state":"idle"} 2026-02-20T21:14:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:14:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:13:58 2026-02-20T21:14:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:14:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:14:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:14:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:14:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:01' 2026-02-20T21:14:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:14:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:14:05.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:14:05.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622037,"replay_state":"idle"} 2026-02-20T21:14:05.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:14:05.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:14:05.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:14:05.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:14:05.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:14:05.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:14:05.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:14:05.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:14:05.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:58' 2026-02-20T21:14:05.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:13:58' 2026-02-20T21:14:05.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:05.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:05.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:05.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:14:05.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:13:58' 2026-02-20T21:14:05.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:05.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:05.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:14:05.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622037,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:13:58cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:14:01' 2026-02-20T21:14:05.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:14:05.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:14:05.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:13:58' 2026-02-20T21:14:05.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:14:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:14:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:14:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:14:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:14:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:05.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:05.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:14:05.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test 2026-02-20T21:14:05.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:05.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:05.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:05.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-20T21:14:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:14:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:14:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:06.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:06.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:06.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:14:06.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:06.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:06.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:14:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:14:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:14:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:14:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:14:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:14:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:14:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:14:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:14:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:14:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:14:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:14:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:14:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:14:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:14:07.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:14:07.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:07.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:14:07.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:14:07.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:07.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:14:07.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:14:07.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:14:07.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:14:07.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:14:07.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:14:07.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:14:07.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:07.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:14:07.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:14:07.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:14:07.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:07.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:14:07.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:14:07.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:14:07.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:14:07.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:14:07.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:14:07.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:14:07.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:14:07.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:14:07.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:07.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:07.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:14:07.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:14:07.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:14:07.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:14:07.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:14:07.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:14:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:14:07.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:14:07.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:14:07.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:14:07.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:14:07.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:14:07.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:14:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:14:09.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:14:09.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:14:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:14:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:14:09.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:14:09.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:14:09.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:09.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:14:09.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:14:09.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:14:09.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:14:09.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:09.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:14:09.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:14:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:14:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:14:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:14:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:14:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:14:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:14:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:14:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:14:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:14:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:14:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:14:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:14:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:14:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:14:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:14:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:14:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:14:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:14:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:14:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:14:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:14:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:14:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:14:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:14:09.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:14:09.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:14:09.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:14:09.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:09.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:09.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:09.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:14:09.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:14:09.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:14:09.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:14:09.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:14:09.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:14:09.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:09.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:14:09.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:09.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:09.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:09.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:14:09.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:09.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:09.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:09.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:14:10.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:14:10.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:10.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:10.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:10.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:14:10.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:10.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:10.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:10.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:10.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:14:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:14:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:14:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:14:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:14:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:14:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:08 2026-02-20T21:14:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:14:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:14:10.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:14:10.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:14:10.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:06' 2026-02-20T21:14:10.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:14:10.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:14:10.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:14:10.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:14:10.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:14:10.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:14:10.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:14:10.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:14:10.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:14:10.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:14:10.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:14:10.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:14:10.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:08' 2026-02-20T21:14:10.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:14:08' 2026-02-20T21:14:10.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:10.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:10.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:10.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:14:10.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:14:08' 2026-02-20T21:14:10.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:10.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:10.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:14:10.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:14:08cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:14:06' 2026-02-20T21:14:10.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:14:10.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:14:10.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:08' 2026-02-20T21:14:10.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:14:10.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:14:10.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:14:10.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:14:10.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:14:10.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:10.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:10.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:14:10.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:10.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:10.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:10.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:14:10.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:10.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:10.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:10.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:14:11.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:14:11.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:11.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:11.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:11.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:14:11.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:11.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:11.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:11.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:11.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:14:11.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:14:11.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:14:11.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:14:11.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:14:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:14:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:06 2026-02-20T21:14:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:14:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:14:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:14:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:14:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:08' 2026-02-20T21:14:11.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:14:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:14:11.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:14:11.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:14:11.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:14:11.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:14:11.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:14:11.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:14:11.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:14:11.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:14:11.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:14:11.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:14:11.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:06' 2026-02-20T21:14:11.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:14:06' 2026-02-20T21:14:11.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:11.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:11.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:11.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:14:11.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:14:06' 2026-02-20T21:14:11.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:11.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:11.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:14:11.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:14:06cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:14:08' 2026-02-20T21:14:11.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:14:11.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:14:11.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:06' 2026-02-20T21:14:11.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:14:11.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:14:11.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:14:11.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:14:11.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:14:11.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:11.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:11.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test 2026-02-20T21:14:11.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:11.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:11.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:11.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:14:11.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-20T21:14:12.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:14:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-20T21:14:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-20T21:14:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:14:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:14:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:14:13.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:14:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:14:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:14:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:14:13.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:14:13.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:14:13.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:14:13.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:13.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:14:13.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:14:13.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:14:13.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:13.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:14:13.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:14:14.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:14:14.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:14.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:14:14.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:14:14.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:14.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:14:14.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:14:14.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:14:14.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:14:14.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:14:14.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:14:14.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:14:14.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:14:14.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:14:14.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:14.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:14:14.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:14.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:14:14.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:14:14.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:14:14.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:14:14.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:14:14.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:14:14.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:14:14.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:14:14.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:14.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:14.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:14:14.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:14:14.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:14:14.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:14:14.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:14:14.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:14:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:14:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:14:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:14:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:14:14.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:14:14.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:14:14.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:14:14.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:14.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:14:14.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:14:14.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:14.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:14:14.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:14:14.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:14:14.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:14:14.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:14:14.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:14:14.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:14:14.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:14:14.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:14:14.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:14.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:14:14.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:14:14.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:14:14.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:14:14.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:14:14.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:14:14.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:14:14.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:14:14.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:14:14.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:14:14.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:14.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:14:14.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:14:14.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:14:14.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:14:14.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:14:14.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:14:14.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:14:14.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:14:14.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:14:14.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:14:14.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:14:14.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:14:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:14:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:14:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:14:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:14:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:14:16.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:14:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:14:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:14:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:14:16.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:14:16.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:14:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:14:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:14:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:14:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:14:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:14:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:14:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:14:16.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:16.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:14:16.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:14:16.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:16.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:14:16.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:14:16.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:14:16.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:14:16.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:14:16.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:14:16.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:14:16.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:14:16.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:14:16.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:16.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:14:16.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:16.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:14:16.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:14:16.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:14:16.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:14:16.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:14:16.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:14:16.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:14:16.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:14:16.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:16.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:14:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:14:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:14:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:14:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:14:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:14:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:14:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:14:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:14:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:14:16.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:14:16.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:14:16.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:14:16.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:16.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:14:16.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:14:16.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:16.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:14:16.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:14:16.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:14:16.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:14:16.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:14:16.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:14:16.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:14:16.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:14:16.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:14:16.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:16.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:14:16.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:14:16.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:14:16.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:14:16.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:14:16.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:14:16.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:14:16.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:14:16.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:14:16.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:14:16.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:16.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:14:16.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:14:16.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:14:16.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:14:16.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:14:16.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:14:16.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:14:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:14:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:14:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:14:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:14:16.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:14:16.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:14:16.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:14:16.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:14:16.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:14:16.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:16.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:14:20.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:14:20.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:20.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:20.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:20.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:14:20.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:14:20.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:14:20.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:14:20.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:14:20.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:20.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:14:20.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:14:20.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:14:20.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:20.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:14:20.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:14:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:14:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:14:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:14:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:14:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:14:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:14:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:14:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:14:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:14:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:14:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:14:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:14:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:14:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:14:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:14:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:14:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:14:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:14:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:14:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:14:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:14:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:14:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:14:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:14:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:14:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:14:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:14:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:14:20.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:14:20.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:14:20.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:14:20.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:14:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:14:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:14:20.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:14:20.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:14:20.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:14:20.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:14:20.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:14:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:14:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:14:20.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:14:20.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:14:20.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:14:20.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:14:20.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:14:20.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:14:20.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:14:20.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:20.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:14:20.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:14:20.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:20.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:14:20.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:14:20.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:20.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:14:20.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:14:20.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-20T21:14:20.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:20.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:20.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:20.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-20T21:14:22.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 105 2026-02-20T21:14:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:14:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:14:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:14:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:14:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:14:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:14:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:14:22.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:14:22.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:14:22.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:14:22.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:14:22.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:14:22.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:14:22.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:22.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:22.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:22.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:14:22.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:14:22.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:14:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:14:22.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=105 2026-02-20T21:14:22.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:14:22.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:14:22.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:14:23.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:14:23.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:23.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:23.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:23.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:14:23.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:14:23.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:14:23.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:14:23.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=105 2026-02-20T21:14:23.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 105 = 105 2026-02-20T21:14:23.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:23.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:14:23.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:23.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:23.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:23.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:14:23.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:23.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:23.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:23.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:14:24.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:14:24.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:24.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:24.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:24.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:14:24.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:24.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:24.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:24.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:24.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:14:24.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:14:24.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:14:24.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:14:24.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:14:24.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:14:24.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:23 2026-02-20T21:14:24.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:14:24.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:14:24.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:14:24.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622051,"remote_snapshot_timestamp":1771622051,"replay_state":"idle"} 2026-02-20T21:14:24.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:21 2026-02-20T21:14:24.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:14:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 104 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.65434b1e-5de2-48fd-9176-767e4e34b203 (peer_uuids:[]) 2026-02-20T21:14:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 105 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.3f4839d1-3b8a-4705-9350-b50afa34163e (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:14:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:14:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:14:24.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:14:24.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:14:24.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:14:24.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:14:24.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:14:24.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:14:24.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:14:24.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:14:24.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:14:24.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:14:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:23' 2026-02-20T21:14:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:14:23' 2026-02-20T21:14:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:14:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:14:23' 2026-02-20T21:14:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:14:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary44044409mirror.0trial1992026-02-20 21:14:23cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622051,"remote_snapshot_timestamp":1771622051,"replay_state":"idle"}2026-02-20 21:14:21' 2026-02-20T21:14:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:14:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:14:24.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:23' 2026-02-20T21:14:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:14:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:14:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:14:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:14:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:14:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-20T21:14:24.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:24.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:24.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:24.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:14:24.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:24.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:24.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:24.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:14:25.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-20T21:14:25.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:25.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:14:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:14:25.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:14:25.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:14:25.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:14:25.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622051,"remote_snapshot_timestamp":1771622051,"replay_state":"idle"} 2026-02-20T21:14:25.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:14:25.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:21 2026-02-20T21:14:25.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:14:25.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:14:25.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:14:25.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:14:25.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:23' 2026-02-20T21:14:25.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:14:25.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:14:25.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:14:25.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622051,"remote_snapshot_timestamp":1771622051,"replay_state":"idle"} 2026-02-20T21:14:25.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:14:25.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:14:25.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:14:25.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:14:25.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:14:25.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:14:25.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:14:25.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:14:25.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:21' 2026-02-20T21:14:25.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-20 21:14:21' 2026-02-20T21:14:25.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:25.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:25.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:25.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:14:25.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:14:21' 2026-02-20T21:14:25.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:25.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:25.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:14:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622051,"remote_snapshot_timestamp":1771622051,"replay_state":"idle"}45294534mirror.1trial1992026-02-20 21:14:21cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:14:23' 2026-02-20T21:14:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:14:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:14:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:21' 2026-02-20T21:14:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:14:25.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:14:25.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:14:25.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:14:25.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:14:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster1 mirror test 2026-02-20T21:14:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-20T21:14:26.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:14:26.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-20T21:14:26.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:26.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:26.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:26.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-20T21:14:26.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:26.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:26.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:26.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:14:26.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:14:26.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:26.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:14:27.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:14:27.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:27.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:27.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:27.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:14:27.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:14:27.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:14:27.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:14:27.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:14:27.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:14:27.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:27.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:14:27.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:14:27.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:14:27.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:27.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:14:27.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:14:27.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:27.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:14:27.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:14:27.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:14:27.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:14:27.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:14:27.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:14:27.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:14:27.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:14:27.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:14:27.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:14:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:14:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:14:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:14:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:14:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:14:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:14:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:14:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:14:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:14:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:14:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:14:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:14:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:14:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:14:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:14:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:14:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:14:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:14:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:14:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:14:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:14:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:14:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:14:27.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:14:27.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:27.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:14:27.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:14:27.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:14:27.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:14:27.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:14:27.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:14:27.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:14:27.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:14:27.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:14:27.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:14:27.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:14:28.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:14:28.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:14:28.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:14:28.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:14:28.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:14:28.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:28.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:14:28.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:28.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:28.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:28.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:14:28.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:28.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:28.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:28.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:14:29.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:14:29.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:29.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:29.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:29.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:14:29.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:29.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:29.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:29.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:29.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:14:29.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:14:29.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:14:29.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:14:29.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:14:29.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:14:29.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:28 2026-02-20T21:14:29.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:14:29.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:14:29.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:14:29.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:14:29.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:27' 2026-02-20T21:14:29.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:14:29.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:14:29.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:14:29.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:14:29.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:14:29.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:14:29.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:14:29.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:14:29.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:14:29.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:14:29.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:14:29.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:14:29.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:28' 2026-02-20T21:14:29.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:14:28' 2026-02-20T21:14:29.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:29.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:29.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:29.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:14:29.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:14:28' 2026-02-20T21:14:29.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:29.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:29.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:14:29.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:14:28cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image demoted2026-02-20 21:14:27' 2026-02-20T21:14:29.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:14:29.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:14:29.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:28' 2026-02-20T21:14:29.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:14:29.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:14:29.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:14:29.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:14:29.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:14:29.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:29.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:29.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:14:29.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:29.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:29.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:29.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:14:29.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:29.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:29.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:29.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:14:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:14:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:14:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:14:30.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:14:30.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:14:30.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:14:30.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:14:30.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:14:30.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:27 2026-02-20T21:14:30.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:14:30.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:14:30.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:14:30.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:14:30.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:28' 2026-02-20T21:14:30.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:14:30.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:14:30.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:14:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image demoted 2026-02-20T21:14:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:14:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:14:30.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:14:30.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:14:30.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:14:30.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:14:30.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:14:30.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:14:30.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:27' 2026-02-20T21:14:30.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:14:27' 2026-02-20T21:14:30.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:30.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:30.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:30.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:14:30.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:14:27' 2026-02-20T21:14:30.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:30.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:30.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:14:30.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image demoted45294534mirror.1trial1992026-02-20 21:14:27cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:14:28' 2026-02-20T21:14:30.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:14:30.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:14:30.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:27' 2026-02-20T21:14:30.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:14:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:14:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:14:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:14:30.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:14:30.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:30.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:30.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster2 mirror test 2026-02-20T21:14:30.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:30.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:30.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:30.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:14:30.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-20T21:14:31.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:14:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:14:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:14:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:14:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:14:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:14:32.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:14:32.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:32.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:32.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:32.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:14:32.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:14:32.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:14:32.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:14:32.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:14:32.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:32.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:14:32.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:14:32.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:14:32.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:14:32.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:32.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:14:32.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:14:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:14:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:14:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:14:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:14:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:14:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:14:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:14:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:14:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:14:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:14:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:14:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:14:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:14:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:14:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:14:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:14:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:14:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:14:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:14:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:14:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:14:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:14:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:14:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:14:32.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:14:32.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:14:32.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:14:32.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:14:32.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:14:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:14:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:14:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:14:32.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:14:32.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:32.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:14:34.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:14:34.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:34.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:34.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:34.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:14:34.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:14:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:14:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:14:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:14:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:14:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:14:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:14:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:14:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:14:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:14:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:14:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:14:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:14:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:14:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:14:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:14:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:14:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:14:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:14:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:14:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:14:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:14:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:14:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:14:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:14:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:14:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:14:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:14:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:14:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:14:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:14:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:14:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:14:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:14:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:14:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:14:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:14:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:14:35.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:14:35.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:14:35.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:14:35.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:14:35.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:14:35.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:14:35.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:14:35.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:35.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:14:35.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:14:35.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:35.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:14:35.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:14:35.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:35.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:14:35.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:14:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:14:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:14:37.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 108 2026-02-20T21:14:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:14:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:14:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:14:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:14:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:14:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:14:37.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:14:37.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:14:37.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:14:37.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:14:37.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:14:37.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:14:37.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:14:37.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:37.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:37.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:37.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:14:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:14:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:14:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:14:37.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=108 2026-02-20T21:14:37.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:14:37.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:14:37.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:14:38.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:14:38.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:38.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:38.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:38.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:14:38.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:14:38.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:14:38.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:14:38.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=108 2026-02-20T21:14:38.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 108 = 108 2026-02-20T21:14:38.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:38.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-20T21:14:38.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:38.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:38.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:38.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:14:38.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:38.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:38.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:38.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:14:39.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-20T21:14:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:14:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:39.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:14:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:14:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:14:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:14:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:14:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:14:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:36 2026-02-20T21:14:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:14:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:14:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:14:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622070,"replay_state":"idle"} 2026-02-20T21:14:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:33 2026-02-20T21:14:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:14:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 107 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.7765f955-2b51-47a0-8bd6-abe6599c668d (peer_uuids:[]) 2026-02-20T21:14:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 108 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.cbdffbb6-94d2-499b-915f-0eb4edfbce3e (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:14:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:14:39.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:14:39.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:14:39.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:14:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:14:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:14:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:14:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:14:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:14:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:14:39.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:14:39.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:14:39.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:36' 2026-02-20T21:14:39.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-20 21:14:36' 2026-02-20T21:14:39.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:39.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:39.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:39.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:14:39.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:14:36' 2026-02-20T21:14:39.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:39.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:39.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:14:39.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:14:36cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622070,"replay_state":"idle"}2026-02-20 21:14:33' 2026-02-20T21:14:39.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:14:39.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:14:39.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:36' 2026-02-20T21:14:39.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:14:39.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:14:39.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:14:39.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:14:39.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:14:39.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:39.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:39.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:14:39.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:39.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:39.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:39.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:14:39.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:39.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:39.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:39.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:14:40.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:14:40.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:40.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:40.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:40.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:14:40.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:40.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:40.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:40.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:40.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:14:40.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:14:40.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:14:40.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:14:40.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622070,"replay_state":"idle"} 2026-02-20T21:14:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:14:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:33 2026-02-20T21:14:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:14:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:14:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:14:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:14:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:36' 2026-02-20T21:14:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:14:40.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:14:40.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:14:40.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622070,"replay_state":"idle"} 2026-02-20T21:14:40.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:14:40.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:14:40.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:14:40.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:14:40.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:14:40.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:14:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:14:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:14:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:33' 2026-02-20T21:14:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:14:33' 2026-02-20T21:14:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:14:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:14:33' 2026-02-20T21:14:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:14:40.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622070,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:14:33cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:14:36' 2026-02-20T21:14:40.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:14:40.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:14:40.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:33' 2026-02-20T21:14:40.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:14:40.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:14:40.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:14:40.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:14:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:14:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:14:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test 2026-02-20T21:14:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:40.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-20T21:14:41.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:14:41.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:14:41.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:41.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:41.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:41.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:14:41.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:41.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:41.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:41.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:14:41.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:14:41.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:41.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:14:42.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:14:42.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:14:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:14:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:14:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:14:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:14:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:14:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:14:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:14:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:14:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:14:42.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:14:42.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:14:42.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:42.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:14:42.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:14:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:14:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:14:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:14:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:14:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:14:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:14:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:14:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:14:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:14:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:14:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:14:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:14:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:14:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:14:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:14:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:14:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:14:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:14:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:14:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:42.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:14:42.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:14:42.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:14:42.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:14:42.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:14:42.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:42.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:42.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:42.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:14:42.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:14:42.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:14:42.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:14:42.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:14:42.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:14:42.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:42.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:14:42.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:42.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:42.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:43.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:14:43.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:43.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:43.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:43.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:14:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:14:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:14:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:14:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:14:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:14:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:14:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:14:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:14:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:43 2026-02-20T21:14:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:14:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:14:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:14:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:14:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:41' 2026-02-20T21:14:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:14:44.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:14:44.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:14:44.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:14:44.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:14:44.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:14:44.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:14:44.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:14:44.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:14:44.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:14:44.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:14:44.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:14:44.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:43' 2026-02-20T21:14:44.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:14:43' 2026-02-20T21:14:44.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:44.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:44.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:44.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:14:44.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:14:43' 2026-02-20T21:14:44.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:44.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:44.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:14:44.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:14:43cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:14:41' 2026-02-20T21:14:44.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:14:44.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:14:44.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:43' 2026-02-20T21:14:44.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:14:44.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:14:44.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:14:44.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:14:44.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:14:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:14:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:44.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:14:44.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:44.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:44.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:44.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:14:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:14:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:14:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:45.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:45.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:14:45.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:14:45.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:14:45.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:14:45.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:14:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:14:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:41 2026-02-20T21:14:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:14:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:14:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:14:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:14:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:43' 2026-02-20T21:14:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:14:45.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:14:45.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:14:45.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:14:45.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:14:45.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:14:45.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:14:45.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:14:45.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:14:45.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:14:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:14:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:14:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:41' 2026-02-20T21:14:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:14:41' 2026-02-20T21:14:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:14:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:14:41' 2026-02-20T21:14:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:45.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:45.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:14:45.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:14:41cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:14:43' 2026-02-20T21:14:45.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:14:45.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:14:45.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:41' 2026-02-20T21:14:45.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:14:45.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:14:45.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:14:45.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:14:45.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:14:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test 2026-02-20T21:14:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:14:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-20T21:14:46.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:14:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-20T21:14:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-20T21:14:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:14:46.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:14:46.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:46.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:14:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:14:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:14:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:14:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:14:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:14:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:14:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:14:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:14:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:14:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:14:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:14:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:14:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:14:47.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:47.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:14:47.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:14:47.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:47.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:14:47.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:14:47.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:14:47.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:14:47.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:14:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:14:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:14:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:14:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:14:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:14:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:14:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:14:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:14:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:14:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:14:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:14:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:14:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:14:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:14:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:14:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:14:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:14:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:14:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:14:47.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:14:47.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:14:47.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:14:47.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:14:47.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:14:47.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:14:47.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:14:47.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:14:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:14:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:14:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:14:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:14:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:14:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:14:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:14:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:14:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:14:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:14:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:14:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:14:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:14:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:14:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:14:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:14:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:14:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:14:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:14:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:14:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:14:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:14:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:14:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:14:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:14:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:14:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:14:48.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:14:48.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:14:48.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:14:48.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:14:48.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:14:48.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:14:48.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:14:48.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:14:48.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:14:48.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:14:48.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:14:48.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:48.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:14:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:14:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:14:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:14:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:14:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:14:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-20T21:14:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-20T21:14:51.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 111 2026-02-20T21:14:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:14:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:14:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:14:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:14:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:14:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:14:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:14:51.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:14:51.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:14:51.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:14:51.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:14:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:14:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:14:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:14:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:14:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:14:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:14:51.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=111 2026-02-20T21:14:51.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:14:51.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:14:51.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:14:52.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:14:52.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:52.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:52.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:52.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:14:52.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:14:52.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:14:52.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:14:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=111 2026-02-20T21:14:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 111 = 111 2026-02-20T21:14:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:14:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:14:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:14:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:14:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:14:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:14:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:14:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:14:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:14:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:14:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:14:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:53 2026-02-20T21:14:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:14:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:14:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:14:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622088,"remote_snapshot_timestamp":1771622088,"replay_state":"idle"} 2026-02-20T21:14:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:51 2026-02-20T21:14:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:14:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 111 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.d600ad0d-d914-4e34-a86c-d9351fce8769 (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:14:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:14:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:14:53.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:14:53.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:14:53.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:14:53.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:14:53.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:14:53.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:14:53.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:14:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:14:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:14:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:14:53.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:53' 2026-02-20T21:14:53.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:14:53' 2026-02-20T21:14:53.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:53.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:53.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:53.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:14:53.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:14:53' 2026-02-20T21:14:53.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:53.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:53.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:14:53.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary44044409mirror.0trial1992026-02-20 21:14:53cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622088,"remote_snapshot_timestamp":1771622088,"replay_state":"idle"}2026-02-20 21:14:51' 2026-02-20T21:14:53.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:14:53.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:14:53.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:53' 2026-02-20T21:14:53.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:14:53.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:14:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:14:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:14:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:14:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:53.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-20T21:14:53.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:53.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:53.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:53.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:14:53.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:53.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:53.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:53.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:14:54.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-20T21:14:54.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:54.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:54.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:54.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:14:54.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:54.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:54.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:54.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:54.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:14:54.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:14:54.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:14:54.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:14:54.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622088,"remote_snapshot_timestamp":1771622088,"replay_state":"idle"} 2026-02-20T21:14:54.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:14:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:51 2026-02-20T21:14:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:14:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:14:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:14:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:14:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:53' 2026-02-20T21:14:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:14:54.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:14:54.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:14:54.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622088,"remote_snapshot_timestamp":1771622088,"replay_state":"idle"} 2026-02-20T21:14:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:14:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:14:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:14:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:14:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:14:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:14:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:14:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:14:54.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:51' 2026-02-20T21:14:54.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-20 21:14:51' 2026-02-20T21:14:54.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:54.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:54.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:54.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:14:54.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:14:51' 2026-02-20T21:14:54.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:54.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:54.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:14:54.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622088,"remote_snapshot_timestamp":1771622088,"replay_state":"idle"}45294534mirror.1trial1992026-02-20 21:14:51cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:14:53' 2026-02-20T21:14:54.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:14:54.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:14:54.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:51' 2026-02-20T21:14:54.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:14:54.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:14:54.353 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:14:54.353 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:14:54.353 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:14:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster1 mirror test 2026-02-20T21:14:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-20T21:14:54.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:14:54.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-20T21:14:54.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:54.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:54.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:54.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-20T21:14:54.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:54.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:54.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:54.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:14:54.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:14:54.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:54.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:14:55.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:14:55.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:55.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:55.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:55.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:14:55.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:14:55.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:14:55.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:14:55.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:14:55.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:55.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:14:55.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:14:55.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:14:55.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:14:55.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:55.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:14:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:14:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:14:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:14:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:14:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:14:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:14:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:14:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:14:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:14:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:14:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:14:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:14:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:14:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:14:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:55.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:14:55.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:14:55.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:14:55.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:14:55.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:14:55.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:14:55.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:14:55.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:14:55.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:55.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:55.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:14:55.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:14:55.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:14:55.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:14:55.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:14:55.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:14:55.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:14:55.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:14:55.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:14:55.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:14:56.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:14:56.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:14:56.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:14:56.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:56.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:14:56.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:14:56.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:14:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:14:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:14:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:14:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:14:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:14:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:14:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:14:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:14:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:14:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:14:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:14:56.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:14:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:14:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:14:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:14:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:14:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:14:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:14:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:14:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:14:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:14:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:14:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:14:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:14:56.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:14:56.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:14:56.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:14:56.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:14:56.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:14:56.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:14:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:14:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:14:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:14:56.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:14:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:14:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:14:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:14:58.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:14:58.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:58.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:58.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:58.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:14:58.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:14:58.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:14:58.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:14:58.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:14:58.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:58.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:14:58.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:14:58.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:14:58.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:14:58.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:58.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:14:58.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:14:58.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:14:58.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:58.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:14:58.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:14:58.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:58.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:14:58.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:14:58.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:14:58.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:14:58.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:14:58.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:14:58.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:14:58.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:14:58.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:14:58.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:58.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:14:58.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:58.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:14:58.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:14:58.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:14:58.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:14:58.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:14:58.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:14:58.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:14:58.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:14:58.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:58.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:14:58.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:14:58.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:14:58.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:14:58.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:14:58.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:14:58.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:14:58.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:14:58.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:14:58.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:14:58.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:14:58.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:14:58.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:14:58.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:14:58.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:58.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:14:58.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:14:58.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:14:58.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:14:58.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:14:58.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:14:58.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:14:58.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:14:58.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:14:58.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:14:58.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:14:58.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:14:58.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:58.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:14:58.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:14:58.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:14:58.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:14:58.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:14:58.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:14:58.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:14:58.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:14:58.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:14:58.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:14:58.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:14:58.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:14:58.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:14:58.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:14:58.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:14:58.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:14:58.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:14:58.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:14:58.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:14:58.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:14:58.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:14:58.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:14:58.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:14:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:14:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:14:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:14:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:14:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:14:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:14:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:58.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:58.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:58.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:14:59.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:14:59.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:59.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:59.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:59.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:14:59.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:59.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:59.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:59.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:59.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:14:59.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:14:59.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:14:59.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:14:59.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:14:59.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:14:59.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:58 2026-02-20T21:14:59.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:14:59.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:14:59.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:14:59.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:14:59.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:56' 2026-02-20T21:14:59.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:14:59.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:14:59.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:14:59.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:14:59.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:14:59.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:14:59.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:14:59.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:14:59.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:14:59.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:14:59.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:14:59.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:14:59.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:58' 2026-02-20T21:14:59.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:14:58' 2026-02-20T21:14:59.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:14:59.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:59.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:59.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:14:59.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:14:58' 2026-02-20T21:14:59.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:14:59.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:14:59.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:14:59.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:14:58cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:14:56' 2026-02-20T21:14:59.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:14:59.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:14:59.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:58' 2026-02-20T21:14:59.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:14:59.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:14:59.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:14:59.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:14:59.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:14:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:14:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:14:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:14:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:14:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:14:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:14:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:14:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:14:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:15:00.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:15:00.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:00.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:00.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:00.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:15:00.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:15:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:15:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:15:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:15:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:15:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:15:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:56 2026-02-20T21:15:00.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:15:00.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:15:00.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:15:00.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:15:00.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:14:58' 2026-02-20T21:15:00.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:15:00.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:15:00.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:15:00.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:15:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:15:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:15:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:15:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:15:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:15:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:15:00.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:15:00.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:15:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:56' 2026-02-20T21:15:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:14:56' 2026-02-20T21:15:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:00.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:00.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:00.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:15:00.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:14:56' 2026-02-20T21:15:00.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:00.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:00.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:15:00.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:14:56cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:14:58' 2026-02-20T21:15:00.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:15:00.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:15:00.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:14:56' 2026-02-20T21:15:00.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:15:00.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:15:00.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:15:00.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:15:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:15:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster2 mirror test 2026-02-20T21:15:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:00.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:15:00.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-20T21:15:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:15:01.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:15:01.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:01.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:01.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:01.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:15:01.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:01.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:01.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:01.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:15:01.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:15:01.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:01.904 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:15:02.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:15:02.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:02.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:02.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:02.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:15:02.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:15:02.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:15:02.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:15:02.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:15:02.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:15:02.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:02.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:15:02.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:15:02.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:15:02.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:02.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:15:02.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:02.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:15:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:15:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:15:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:15:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:15:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:15:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:15:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:15:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:15:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:15:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:15:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:15:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:15:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:15:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:15:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:15:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:15:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:15:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:15:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:15:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:15:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:15:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:15:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:02.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:02.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:02.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:02.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:15:02.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:15:02.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:15:02.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:15:02.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:15:02.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:15:02.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:02.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:15:04.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:15:04.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:04.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:04.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:04.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:15:04.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:15:04.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:15:04.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:15:04.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:15:04.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:15:04.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:04.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:15:04.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:15:04.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:15:04.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:04.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:15:04.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:04.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:15:04.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:04.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:15:04.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:15:04.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:04.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:15:04.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:04.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:15:04.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:15:04.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:15:04.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:15:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:15:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:15:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:15:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:15:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:15:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:15:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:15:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:04.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:15:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:15:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:15:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:15:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:15:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:15:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:15:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:15:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:04.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:04.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:15:05.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:15:05.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:15:05.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:15:05.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:15:05.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:15:05.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:15:05.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:15:05.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:05.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:15:05.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:15:05.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:05.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:15:05.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:15:05.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:05.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:15:05.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:15:05.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:15:05.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:05.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:05.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:05.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:15:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 114 2026-02-20T21:15:08.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:15:08.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:15:08.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:08.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:15:08.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:15:08.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:08.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:08.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:15:08.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:08.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:15:08.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:15:08.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:15:08.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:08.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:15:08.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:15:08.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:08.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:08.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:08.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:15:08.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:15:08.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:15:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:15:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=114 2026-02-20T21:15:08.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:15:08.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:15:08.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:15:08.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:15:08.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:08.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:08.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:08.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:15:08.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:15:08.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:15:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:15:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=114 2026-02-20T21:15:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 114 = 114 2026-02-20T21:15:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-20T21:15:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:15:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:15:09.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-20T21:15:09.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:09.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:09.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:09.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:15:09.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:09.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:09.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:09.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:09.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:15:09.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:15:09.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:15:09.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:15:09.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:15:09.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:15:09.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:06 2026-02-20T21:15:09.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:15:09.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:15:09.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:15:09.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622100,"replay_state":"idle"} 2026-02-20T21:15:09.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:03 2026-02-20T21:15:09.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:15:09.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 114 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.ad2bff45-1b51-4485-be29-ccde182ef288 (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:15:09.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:15:09.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:15:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:15:09.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:15:09.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:15:09.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:15:09.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:15:09.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:15:09.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:15:09.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:15:09.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:15:09.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:15:09.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:06' 2026-02-20T21:15:09.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-20 21:15:06' 2026-02-20T21:15:09.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:09.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:09.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:09.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:15:09.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:15:06' 2026-02-20T21:15:09.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:09.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:09.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:15:09.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:15:06cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622100,"replay_state":"idle"}2026-02-20 21:15:03' 2026-02-20T21:15:09.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:15:09.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:15:09.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:06' 2026-02-20T21:15:09.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:15:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:15:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:15:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:15:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:15:09.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:15:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:15:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:15:10.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:15:10.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:10.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:10.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:10.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:15:10.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:10.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:10.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:10.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:10.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:15:10.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:15:10.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:15:10.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:15:10.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622100,"replay_state":"idle"} 2026-02-20T21:15:10.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:15:10.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:03 2026-02-20T21:15:10.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:15:10.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:15:10.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:15:10.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:15:10.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:06' 2026-02-20T21:15:10.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:15:10.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:15:10.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:15:10.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622100,"replay_state":"idle"} 2026-02-20T21:15:10.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:15:10.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:15:10.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:15:10.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:15:10.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:15:10.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:15:10.499 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:15:10.499 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:15:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:03' 2026-02-20T21:15:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:15:03' 2026-02-20T21:15:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:15:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:15:03' 2026-02-20T21:15:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:15:10.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622100,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:15:03cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:15:06' 2026-02-20T21:15:10.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:15:10.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:15:10.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:03' 2026-02-20T21:15:10.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:15:10.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:15:10.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:15:10.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:15:10.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:15:10.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:10.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:10.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:15:10.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test 2026-02-20T21:15:10.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:10.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:10.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:10.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-20T21:15:11.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:15:11.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:15:11.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:11.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:11.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:11.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:15:11.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:11.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:11.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:11.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:15:11.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:15:11.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:11.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:15:12.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:15:12.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:12.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:12.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:12.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:15:12.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:15:12.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:15:12.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:15:12.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:15:12.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:12.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:15:12.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:15:12.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:15:12.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:12.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:15:12.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:15:12.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:12.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:15:12.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:12.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:15:12.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:15:12.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:12.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:15:12.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:12.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:15:12.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:15:12.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:15:12.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:15:12.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:15:12.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:12.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:15:12.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:15:12.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:15:12.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:12.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:12.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:15:12.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:15:12.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:15:12.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:12.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:15:12.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:15:12.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:15:12.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:15:12.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:12.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:12.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:15:12.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:15:12.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:15:12.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:15:12.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:12.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:12.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:12.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:12.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:15:12.549 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:15:12.550 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:15:12.550 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:15:12.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:15:12.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:15:12.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:15:12.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:15:12.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:12.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:15:14.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:15:14.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:14.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:14.554 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:14.554 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:15:14.554 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:15:14.554 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:15:14.555 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:15:14.555 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:15:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:15:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:15:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:15:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:15:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:14.556 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:15:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:14.557 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:15:14.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:14.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:15:14.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:15:14.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:14.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:15:14.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:14.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:15:14.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:15:14.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:15:14.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:15:14.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:15:14.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:14.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:15:14.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:15:14.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:15:14.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:14.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:14.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:15:14.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:15:14.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:15:14.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:14.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:15:14.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:15:14.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:15:14.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:15:14.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:14.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:14.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:15:14.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:15:14.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:15:14.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:15:14.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:15:14.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:15:14.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:15:14.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:15:14.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:15:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:15:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:15:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:15:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:15:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:15:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:15:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:15:15.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:15:15.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:15:15.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:15:15.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:15:15.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:15:15.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:13 2026-02-20T21:15:15.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:15:15.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:15:15.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:15:15.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:15:15.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:11' 2026-02-20T21:15:15.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:15:15.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:15:15.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:15:15.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:15:15.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:15:15.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:15:15.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:15:15.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:15:15.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:15:15.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:15:15.656 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:15:15.656 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:15:15.657 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:13' 2026-02-20T21:15:15.657 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:15:13' 2026-02-20T21:15:15.657 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:15.657 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:15.657 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:15.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:15:15.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:15:13' 2026-02-20T21:15:15.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:15.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:15.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:15:15.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:15:13cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:15:11' 2026-02-20T21:15:15.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:15:15.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:15:15.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:13' 2026-02-20T21:15:15.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:15:15.692 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:15:15.692 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:15:15.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:15:15.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:15:15.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:15.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:15.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:15:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:15:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:15:16.696 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:15:16.696 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:16.696 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:16.696 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:16.696 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:15:16.696 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:16.696 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:16.696 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:16.697 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:16.697 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:15:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:15:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:15:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:15:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:15:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:15:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:11 2026-02-20T21:15:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:15:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:15:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:15:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:15:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:13' 2026-02-20T21:15:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:15:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:15:16.726 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:15:16.726 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:15:16.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:15:16.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:15:16.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:15:16.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:15:16.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:15:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:15:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:15:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:15:16.732 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:11' 2026-02-20T21:15:16.732 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:15:11' 2026-02-20T21:15:16.732 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:16.732 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:16.732 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:16.732 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:15:16.732 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:15:11' 2026-02-20T21:15:16.732 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:15:16.762 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:15:11cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:15:13' 2026-02-20T21:15:16.762 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:15:16.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:15:16.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:11' 2026-02-20T21:15:16.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:15:16.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:15:16.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:15:16.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:15:16.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:15:16.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:16.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:16.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test 2026-02-20T21:15:16.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:16.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:16.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:16.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:15:16.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-20T21:15:17.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:15:17.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-20T21:15:17.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:17.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:17.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:17.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-20T21:15:17.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:17.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:17.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:17.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:15:17.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:15:17.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:17.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:15:18.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:15:18.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:18.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:18.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:18.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:15:18.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:15:18.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:15:18.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:15:18.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:15:18.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:15:18.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:18.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:15:18.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:15:18.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:15:18.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:18.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:15:18.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:15:18.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:15:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:15:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:15:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:15:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:15:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:15:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:15:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:15:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:15:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:15:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:15:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:15:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:15:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:15:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:15:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:15:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:15:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:15:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:15:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:15:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:15:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:15:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:15:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:15:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:15:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:15:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:15:18.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:15:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:15:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:15:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:15:19.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:15:19.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:15:19.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:15:19.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:19.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:15:19.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:15:19.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:19.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:15:19.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:15:19.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:15:19.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:15:19.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:15:19.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:15:19.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:15:19.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:15:19.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:15:19.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:19.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:15:19.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:15:19.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:15:19.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:15:19.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:15:19.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:15:19.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:15:19.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:15:19.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:15:19.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:15:19.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:19.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:15:19.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:15:19.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:15:19.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:15:19.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:15:19.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:15:19.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:15:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:15:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:15:19.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:15:19.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:15:19.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:15:19.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:15:19.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:15:19.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:15:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:15:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:15:21.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:15:21.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:21.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:21.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:21.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:15:21.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:15:21.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:15:21.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:15:21.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:15:21.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:15:21.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:21.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:15:21.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:15:21.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:15:21.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:21.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:15:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:15:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:15:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:15:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:15:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:15:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:15:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:15:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:15:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:15:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:15:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:15:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:15:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:15:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:15:21.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:15:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:15:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:15:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:15:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:15:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:15:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:15:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:15:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:15:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:15:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:15:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:15:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:15:21.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:15:21.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:15:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:15:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:15:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:15:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:15:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:15:21.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:15:21.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:21.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:15:21.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:15:21.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:21.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:15:21.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:15:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:15:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:15:21.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:15:21.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:15:21.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:15:21.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:15:21.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:15:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:15:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:15:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:15:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:15:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:15:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:15:25.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:15:25.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:25.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:25.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:25.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:15:25.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:15:25.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:15:25.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:15:25.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:15:25.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:15:25.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:25.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:15:25.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:15:25.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:15:25.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:25.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:15:25.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:15:25.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:15:25.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:25.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:15:25.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:15:25.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:25.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:15:25.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:15:25.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:15:25.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:15:25.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:15:25.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:15:25.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:15:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:15:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:15:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:15:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:15:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:15:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:15:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:15:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:15:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:15:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:15:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:15:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:15:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:15:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:15:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:15:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:15:25.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:15:25.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:15:25.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:15:25.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:15:25.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:15:25.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:15:25.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:15:25.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:15:25.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:25.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:15:25.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:15:25.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:25.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:15:25.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:15:25.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:15:25.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:15:25.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:15:25.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:15:25.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:15:25.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:15:25.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:15:25.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:25.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:15:25.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:15:25.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:15:25.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:15:25.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:15:25.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:15:25.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:15:25.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:15:25.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:15:25.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:15:25.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:15:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:15:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:15:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:15:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:15:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:15:25.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:15:25.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:15:25.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:15:25.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:15:25.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:15:25.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:15:25.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:15:25.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:15:25.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:15:25.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:15:25.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:15:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:15:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:15:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:15:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:15:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:15:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:15:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:15:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-20T21:15:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-20T21:15:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 117 2026-02-20T21:15:28.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:15:28.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:15:28.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:28.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:15:28.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:15:28.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:15:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:15:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:15:28.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:28.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:15:28.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:15:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:15:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:15:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:15:28.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:15:28.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:15:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=117 2026-02-20T21:15:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:15:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:15:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:15:29.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:15:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:15:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:15:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:15:29.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:15:29.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=117 2026-02-20T21:15:29.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 117 = 117 2026-02-20T21:15:29.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:29.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:15:29.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:29.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:15:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:15:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:15:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:30.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:30.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:15:30.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:30.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:30.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:30.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:30.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:15:30.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:15:30.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:15:30.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:15:30.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:15:30.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:15:30.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:28 2026-02-20T21:15:30.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:15:30.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:15:30.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:15:30.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622116,"replay_state":"idle"} 2026-02-20T21:15:30.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:21 2026-02-20T21:15:30.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:15:30.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 117 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.a1a469ff-f856-4635-b7ad-ca6a1f7a78de (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:15:30.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:15:30.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:15:30.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:15:30.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:15:30.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:15:30.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:15:30.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:15:30.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:15:30.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:15:30.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:15:30.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:15:30.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:15:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:28' 2026-02-20T21:15:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:15:28' 2026-02-20T21:15:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:15:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:15:28' 2026-02-20T21:15:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:15:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary44044409mirror.0trial1992026-02-20 21:15:28cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622116,"replay_state":"idle"}2026-02-20 21:15:21' 2026-02-20T21:15:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:15:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:15:30.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:28' 2026-02-20T21:15:30.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:15:30.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:15:30.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:15:30.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:15:30.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:15:30.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:30.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:30.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-20T21:15:30.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:30.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:30.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:30.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:15:30.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:30.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:30.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:30.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:15:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-20T21:15:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:15:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:15:31.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:15:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:15:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:15:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622116,"replay_state":"idle"} 2026-02-20T21:15:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:15:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:21 2026-02-20T21:15:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:15:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:15:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:15:31.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:15:31.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:28' 2026-02-20T21:15:31.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:15:31.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:15:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:15:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622116,"replay_state":"idle"} 2026-02-20T21:15:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:15:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:15:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:15:31.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:15:31.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:15:31.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:15:31.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:15:31.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:15:31.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:21' 2026-02-20T21:15:31.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-20 21:15:21' 2026-02-20T21:15:31.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:31.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:31.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:31.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:15:31.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:15:21' 2026-02-20T21:15:31.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:31.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:31.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:15:31.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622116,"replay_state":"idle"}45294534mirror.1trial1992026-02-20 21:15:21cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:15:28' 2026-02-20T21:15:31.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:15:31.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:15:31.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:21' 2026-02-20T21:15:31.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:15:31.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:15:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:15:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:15:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:15:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:31.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster1 mirror test 2026-02-20T21:15:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-20T21:15:31.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:15:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-20T21:15:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-20T21:15:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:15:31.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:15:31.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:31.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:15:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:15:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:15:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:15:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:15:32.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:15:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:15:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:15:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:15:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:15:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:15:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:15:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:15:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:15:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:15:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:15:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:15:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:15:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:15:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:15:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:15:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:15:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:15:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:15:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:15:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:15:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:15:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:15:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:15:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:15:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:15:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:15:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:15:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:15:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:15:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:15:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:15:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:15:32.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:15:32.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:15:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:15:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:15:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:15:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:15:33.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:15:33.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:15:33.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:15:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:15:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:15:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:15:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:15:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:15:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:15:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:15:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:15:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:15:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:15:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:15:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:15:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:15:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:15:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:15:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:15:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:15:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:15:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:15:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:15:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:15:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:15:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:15:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:15:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:15:33.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:15:33.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:15:33.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:15:33.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:15:33.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:15:33.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:15:33.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:15:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:15:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:15:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:15:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:15:33.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:15:33.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:15:33.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:15:33.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:33.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:15:35.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:15:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:15:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:15:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:15:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:15:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:15:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:15:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:15:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:15:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:15:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:15:35.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:15:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:15:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:15:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:15:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:15:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:15:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:15:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:15:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:15:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:15:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:15:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:15:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:15:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:15:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:15:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:15:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:15:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:15:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:15:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:15:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:15:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:15:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:15:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:15:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:15:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:15:35.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:15:35.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:15:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:15:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:15:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:15:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:15:35.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:15:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:15:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:15:35.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:35.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:15:35.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:15:35.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:35.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:15:35.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:15:35.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:15:35.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:15:35.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:15:35.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:15:35.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:15:35.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:15:35.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:15:35.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:35.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:15:35.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:15:35.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:15:35.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:15:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:15:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:15:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:15:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:15:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:15:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:15:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:15:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:15:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:15:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:15:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:15:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:15:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:15:35.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:15:35.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:15:35.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:15:35.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:15:35.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:15:35.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:15:35.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:15:35.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:15:35.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:15:35.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:35.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:15:35.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:35.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:35.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:35.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:15:35.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:35.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:35.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:35.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:15:36.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:15:36.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:36.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:36.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:36.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:15:36.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:36.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:36.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:36.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:36.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:15:36.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:15:36.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:15:36.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:15:36.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:15:36.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:15:36.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:33 2026-02-20T21:15:36.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:15:36.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:15:36.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:15:36.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:15:36.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:33' 2026-02-20T21:15:36.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:15:36.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:15:36.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:15:36.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:15:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:15:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:15:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:15:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:15:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:15:36.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:15:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:15:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:15:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:33' 2026-02-20T21:15:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:15:33' 2026-02-20T21:15:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:36.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:36.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:36.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:15:36.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:15:33' 2026-02-20T21:15:36.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:36.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:36.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:15:36.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:15:33cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image demoted2026-02-20 21:15:33' 2026-02-20T21:15:36.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:15:36.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:15:36.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:33' 2026-02-20T21:15:36.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:15:36.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:15:36.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:15:36.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:15:36.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:15:36.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:15:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:15:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:36.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:15:37.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:15:37.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:37.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:37.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:37.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:15:37.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:37.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:37.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:37.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:37.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:15:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:15:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:15:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:15:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:15:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:15:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:36 2026-02-20T21:15:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:15:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:15:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:15:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:15:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:33' 2026-02-20T21:15:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:15:37.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:15:37.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:15:37.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:15:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:15:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:15:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:15:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:15:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:15:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:15:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:15:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:15:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:36' 2026-02-20T21:15:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:15:36' 2026-02-20T21:15:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:37.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:15:37.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:15:36' 2026-02-20T21:15:37.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:37.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:37.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:15:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:15:36cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:15:33' 2026-02-20T21:15:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:15:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:15:37.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:36' 2026-02-20T21:15:37.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:15:37.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:15:37.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:15:37.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:15:37.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:15:37.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:37.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster2 mirror test 2026-02-20T21:15:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:15:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-20T21:15:38.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:15:38.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:15:38.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:15:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:15:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:15:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:15:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:15:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:15:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:15:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:15:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:15:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:15:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:15:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:15:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:15:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:15:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:15:39.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:39.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:15:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:39.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:39.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:39.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:39.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:15:39.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:15:39.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:15:39.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:15:39.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:15:39.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:15:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:15:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:15:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:15:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:15:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:15:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:15:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:15:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:15:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:15:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:39.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:15:42.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 120 2026-02-20T21:15:42.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:15:42.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:15:42.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:42.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:15:42.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:15:42.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:42.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:42.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:15:42.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:42.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:15:42.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:15:42.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:15:42.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:15:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:15:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:15:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:15:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:15:42.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:15:42.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=120 2026-02-20T21:15:42.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:15:42.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:15:42.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:15:43.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:15:43.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:43.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:43.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:43.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:15:43.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:15:43.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:15:43.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:15:43.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=120 2026-02-20T21:15:43.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 120 = 120 2026-02-20T21:15:43.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-20T21:15:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:15:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:15:44.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-20T21:15:44.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:44.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:44.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:44.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:15:44.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:44.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:44.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:44.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:44.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:15:44.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:15:44.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:15:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:15:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:15:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:15:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:41 2026-02-20T21:15:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:15:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:15:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:15:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622137,"replay_state":"idle"} 2026-02-20T21:15:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:38 2026-02-20T21:15:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:15:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 120 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.b4088df0-578c-4d55-abd4-d64293fecc14 (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:15:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:15:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:15:44.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:15:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:15:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:15:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:15:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:15:44.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:15:44.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:15:44.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:15:44.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:15:44.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:15:44.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:41' 2026-02-20T21:15:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-20 21:15:41' 2026-02-20T21:15:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:15:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:15:41' 2026-02-20T21:15:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:44.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:15:44.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:15:41cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622137,"replay_state":"idle"}2026-02-20 21:15:38' 2026-02-20T21:15:44.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:15:44.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:15:44.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:41' 2026-02-20T21:15:44.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:15:44.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:15:44.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:15:44.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:15:44.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:15:44.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:44.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:44.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:15:44.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:44.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:44.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:44.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:15:44.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:44.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:44.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:44.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:15:45.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:15:45.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:45.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:45.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:45.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:15:45.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:45.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:45.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:45.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:45.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:15:45.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:15:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:15:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:15:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622137,"replay_state":"idle"} 2026-02-20T21:15:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:15:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:38 2026-02-20T21:15:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:15:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:15:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:15:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:15:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:41' 2026-02-20T21:15:45.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:15:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:15:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:15:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622137,"replay_state":"idle"} 2026-02-20T21:15:45.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:15:45.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:15:45.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:15:45.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:15:45.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:15:45.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:15:45.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:15:45.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:15:45.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:38' 2026-02-20T21:15:45.499 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:15:38' 2026-02-20T21:15:45.499 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:45.499 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:45.499 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:45.499 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:15:45.499 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:15:38' 2026-02-20T21:15:45.499 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:45.499 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:45.499 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:15:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622137,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:15:38cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:15:41' 2026-02-20T21:15:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:15:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:15:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:38' 2026-02-20T21:15:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:15:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:15:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:15:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:15:45.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:15:45.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:15:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test 2026-02-20T21:15:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-20T21:15:46.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:15:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:15:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:46.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:15:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:15:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:15:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:46.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:15:47.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:15:47.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:47.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:47.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:47.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:15:47.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:15:47.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:15:47.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:15:47.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:15:47.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:15:47.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:47.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:15:47.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:15:47.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:15:47.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:47.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:15:47.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:47.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:15:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:15:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:15:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:15:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:15:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:15:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:15:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:15:47.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:15:47.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:47.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:15:47.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:15:47.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:15:47.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:47.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:47.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:15:47.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:15:47.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:15:47.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:47.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:15:47.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:15:47.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:15:47.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:15:47.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:47.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:47.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:15:47.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:15:47.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:15:47.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:15:47.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:47.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:47.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:47.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:47.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:15:47.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:15:47.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:15:47.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:15:47.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:15:47.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:15:47.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:15:47.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:15:47.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:47.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:15:49.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:15:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:15:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:15:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:15:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:15:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:15:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:15:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:15:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:15:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:15:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:15:49.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:49.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:15:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:15:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:15:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:15:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:15:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:15:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:15:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:15:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:15:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:15:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:15:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:15:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:15:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:15:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:15:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:15:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:15:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:15:49.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:15:49.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:49.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:49.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:15:49.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:15:49.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:15:49.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:15:49.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:15:49.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:15:49.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:15:49.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:15:49.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:15:49.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:15:49.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:49.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:15:49.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:49.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:49.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:49.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:15:49.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:49.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:49.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:49.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:15:50.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:15:50.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:50.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:50.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:50.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:15:50.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:50.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:50.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:50.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:50.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:15:50.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:15:50.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:15:50.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:15:50.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:15:50.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:15:50.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:48 2026-02-20T21:15:50.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:15:50.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:15:50.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:15:50.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:15:50.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:46' 2026-02-20T21:15:50.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:15:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:15:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:15:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:15:50.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:15:50.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:15:50.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:15:50.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:15:50.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:15:50.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:15:50.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:15:50.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:15:50.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:48' 2026-02-20T21:15:50.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:15:48' 2026-02-20T21:15:50.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:50.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:50.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:50.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:15:50.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:15:48' 2026-02-20T21:15:50.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:50.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:50.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:15:50.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:15:48cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:15:46' 2026-02-20T21:15:50.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:15:50.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:15:50.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:48' 2026-02-20T21:15:50.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:15:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:15:50.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:15:50.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:15:50.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:15:50.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:50.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:15:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:15:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:15:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:15:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:15:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:15:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:15:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:15:51.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:15:51.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:15:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:15:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:15:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:15:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:46 2026-02-20T21:15:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:15:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:15:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:15:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:15:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:48' 2026-02-20T21:15:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:15:51.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:15:51.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:15:51.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:15:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:15:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:15:51.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:15:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:15:51.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:15:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:15:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:15:51.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:15:51.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:46' 2026-02-20T21:15:51.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:15:46' 2026-02-20T21:15:51.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:51.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:51.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:51.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:15:51.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:15:46' 2026-02-20T21:15:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:15:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:15:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:15:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:15:46cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:15:48' 2026-02-20T21:15:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:15:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:15:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:46' 2026-02-20T21:15:51.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:15:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:15:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:15:51.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:15:51.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:15:51.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:51.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:15:51.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test 2026-02-20T21:15:51.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:15:51.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:51.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:51.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:15:51.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-20T21:15:52.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:15:53.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-20T21:15:53.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:53.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:53.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:53.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-20T21:15:53.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:53.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:53.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:53.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:15:53.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:15:53.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:53.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:15:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:15:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:15:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:15:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:15:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:15:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:15:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:15:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:15:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:15:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:15:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:15:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:15:54.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:15:54.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:54.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:15:54.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:15:54.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:54.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:15:54.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:15:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:15:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:15:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:15:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:15:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:15:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:15:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:15:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:15:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:15:54.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:15:54.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:15:54.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:15:54.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:15:54.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:15:54.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:15:54.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:15:54.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:54.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:54.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:15:54.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:15:54.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:15:54.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:15:54.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:15:54.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:15:54.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:15:54.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:15:54.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:15:54.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:15:54.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:15:54.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:15:54.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:15:54.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:54.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:15:54.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:15:54.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:54.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:15:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:15:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:15:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:15:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:15:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:15:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:15:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:15:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:15:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:15:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:15:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:15:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:15:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:15:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:15:54.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:15:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:15:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:15:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:15:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:15:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:15:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:15:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:15:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:15:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:15:54.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:15:54.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:15:54.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:15:54.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:15:54.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:15:54.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:15:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:15:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:15:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:15:54.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:15:54.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:54.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:15:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:15:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:15:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:15:56.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:15:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:15:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:15:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:15:56.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:15:56.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:15:56.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:15:56.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:56.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:15:56.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:15:56.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:15:56.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:56.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:15:56.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:15:56.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:15:56.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:56.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:15:56.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:15:56.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:56.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:15:56.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:15:56.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:15:56.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:15:56.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:15:56.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:15:56.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:15:56.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:15:56.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:15:56.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:56.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:15:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:15:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:15:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:15:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:15:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:15:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:15:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:15:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:15:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:15:56.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:15:56.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:15:56.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:15:56.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:15:56.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:15:56.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:15:56.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:15:56.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:15:56.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:15:56.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:15:56.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:15:56.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:15:56.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:15:56.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:56.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:15:56.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:15:56.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:15:56.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:15:56.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:15:56.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:15:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:15:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:15:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:15:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:15:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:15:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:15:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:15:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:15:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:15:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:15:56.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:15:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:15:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:15:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:15:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:15:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:15:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:15:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:15:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:15:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:15:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:15:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:15:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:15:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:15:56.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:15:56.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:15:56.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:15:56.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:15:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:15:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:15:56.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:15:56.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:15:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:15:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:16:00.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:16:00.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:00.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:00.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:00.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:16:00.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:16:00.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:16:00.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:16:00.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:16:00.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:16:00.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:00.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:16:00.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:16:00.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:16:00.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:00.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:16:00.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:16:00.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:16:00.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:00.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:16:00.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:16:00.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:00.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:16:00.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:16:00.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:16:00.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:16:00.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:16:00.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:16:00.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:16:00.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:16:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:16:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:16:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:16:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:16:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:16:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:16:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:16:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:16:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:16:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:16:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:16:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:16:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:16:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:16:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:16:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:16:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:16:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:16:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:16:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:16:00.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:16:00.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:16:00.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:16:00.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:00.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:16:00.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:16:00.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:16:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:16:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:16:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:16:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:16:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:16:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:16:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:16:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:16:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:16:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:16:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:16:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:16:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:16:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:16:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:16:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:16:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:16:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:16:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:16:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:16:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:16:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:16:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:16:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:16:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:16:00.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:16:00.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:16:00.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:16:00.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:16:00.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:16:00.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:16:00.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:16:00.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:16:00.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:16:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:16:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:16:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:16:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:16:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:16:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:16:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:16:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:16:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-20T21:16:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:00.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:00.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:00.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-20T21:16:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 123 2026-02-20T21:16:02.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:16:02.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:16:02.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:02.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:16:02.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:16:02.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:02.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:16:02.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:16:02.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:02.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:16:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:16:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:16:02.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:16:02.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:02.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:02.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:02.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:16:02.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:16:02.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:16:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:16:02.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=123 2026-02-20T21:16:02.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:16:02.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:02.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:16:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:16:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:03.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:16:03.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:16:03.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:16:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:16:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=123 2026-02-20T21:16:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 123 = 123 2026-02-20T21:16:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:16:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:16:03.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:03.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:03.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:03.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:04.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:16:04.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:04.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:04.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:04.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:16:04.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:04.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:04.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:04.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:04.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:16:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:16:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:16:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:16:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:16:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:16:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:03 2026-02-20T21:16:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:16:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:16:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:16:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622151,"replay_state":"idle"} 2026-02-20T21:16:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:56 2026-02-20T21:16:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:16:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 122 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.13c422d5-f57d-4ad8-b3a7-ae07edb55a54 (peer_uuids:[]) 2026-02-20T21:16:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 123 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.84362ef4-30f6-49f3-aeae-16f760b6ee3f (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:16:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:16:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:16:04.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:16:04.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:16:04.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:16:04.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:16:04.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:16:04.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:16:04.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:16:04.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:16:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:16:04.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:16:04.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:03' 2026-02-20T21:16:04.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:16:03' 2026-02-20T21:16:04.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:04.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:04.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:04.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:16:04.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:16:03' 2026-02-20T21:16:04.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:04.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:04.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:16:04.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary44044409mirror.0trial1992026-02-20 21:16:03cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622151,"replay_state":"idle"}2026-02-20 21:15:56' 2026-02-20T21:16:04.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:16:04.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:16:04.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:03' 2026-02-20T21:16:04.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:16:04.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:16:04.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:16:04.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:16:04.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:16:04.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:04.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:04.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-20T21:16:04.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:04.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:04.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:04.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:16:04.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:04.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:04.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:04.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:05.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-20T21:16:05.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:05.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:05.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:05.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:16:05.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:05.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:05.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:05.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:05.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:16:05.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:16:05.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:16:05.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:16:05.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622151,"replay_state":"idle"} 2026-02-20T21:16:05.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:16:05.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:15:56 2026-02-20T21:16:05.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:16:05.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:16:05.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:16:05.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:16:05.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:03' 2026-02-20T21:16:05.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:16:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:16:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:16:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622151,"replay_state":"idle"} 2026-02-20T21:16:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:16:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:16:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:16:05.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:16:05.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:16:05.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:16:05.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:16:05.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:16:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:56' 2026-02-20T21:16:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-20 21:15:56' 2026-02-20T21:16:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:16:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:15:56' 2026-02-20T21:16:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:05.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:05.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:16:05.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622151,"replay_state":"idle"}45294534mirror.1trial1992026-02-20 21:15:56cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:16:03' 2026-02-20T21:16:05.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:16:05.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:16:05.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:15:56' 2026-02-20T21:16:05.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:16:05.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:16:05.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:16:05.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:16:05.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:16:05.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster1 mirror test 2026-02-20T21:16:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:05.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-20T21:16:06.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:16:06.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-20T21:16:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-20T21:16:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:16:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:16:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:06.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:16:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:16:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:16:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:16:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:16:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:16:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:16:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:16:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:16:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:16:07.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:16:07.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:16:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:16:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:16:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:16:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:16:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:16:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:16:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:16:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:16:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:16:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:16:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:16:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:16:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:16:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:16:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:16:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:16:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:16:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:16:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:16:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:16:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:16:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:16:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:16:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:16:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:16:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:16:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:16:07.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:16:07.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:16:07.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:16:07.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:16:07.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:16:07.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:16:07.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:16:07.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:07.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:16:07.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:16:07.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:07.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:16:07.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:16:07.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:16:07.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:16:07.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:16:07.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:16:07.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:16:07.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:16:07.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:16:07.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:07.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:16:07.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:16:07.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:16:07.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:16:07.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:16:07.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:16:07.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:16:07.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:16:07.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:16:07.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:16:07.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:07.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:16:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:16:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:16:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:16:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:16:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:16:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:16:07.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:16:07.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:16:07.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:16:07.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:16:08.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:16:08.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:16:08.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:16:08.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:16:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:16:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:16:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:16:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:09.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:16:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:16:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:16:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:16:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:16:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:16:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:08 2026-02-20T21:16:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:16:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:16:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:16:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:16:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:07' 2026-02-20T21:16:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:16:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:16:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:16:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:16:09.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:16:09.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:16:09.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:16:09.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:16:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:16:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:16:09.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:16:09.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:16:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:08' 2026-02-20T21:16:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:16:08' 2026-02-20T21:16:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:16:08' 2026-02-20T21:16:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:09.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:16:09.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:16:08cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image demoted2026-02-20 21:16:07' 2026-02-20T21:16:09.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:16:09.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:16:09.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:08' 2026-02-20T21:16:09.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:16:09.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:16:09.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:16:09.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:16:09.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:16:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:16:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:16:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:16:10.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:16:10.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:16:10.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:16:10.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:16:10.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:16:10.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:07 2026-02-20T21:16:10.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:16:10.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:16:10.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:16:10.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:16:10.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:08' 2026-02-20T21:16:10.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:16:10.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:16:10.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:16:10.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image demoted 2026-02-20T21:16:10.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:16:10.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:16:10.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:16:10.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:16:10.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:16:10.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:16:10.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:16:10.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:16:10.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:07' 2026-02-20T21:16:10.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:16:07' 2026-02-20T21:16:10.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:10.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:10.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:10.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:10.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:16:07' 2026-02-20T21:16:10.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:10.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:10.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:16:10.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image demoted45294534mirror.1trial1992026-02-20 21:16:07cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:16:08' 2026-02-20T21:16:10.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:16:10.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:16:10.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:07' 2026-02-20T21:16:10.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:16:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:16:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:16:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:16:10.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:16:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster2 mirror test 2026-02-20T21:16:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:16:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-20T21:16:11.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:16:11.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:16:11.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:11.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:11.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:11.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:16:11.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:11.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:11.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:11.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:16:11.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:16:11.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:11.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:16:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:16:12.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:16:12.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:16:12.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:16:12.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:16:12.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:12.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:16:12.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:16:12.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:16:12.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:16:12.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:12.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:16:12.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:16:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:16:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:16:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:16:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:16:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:16:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:16:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:16:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:16:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:16:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:16:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:16:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:16:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:16:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:16:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:16:12.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:16:12.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:16:12.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:16:12.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:12.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:12.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:16:12.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:16:12.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:16:12.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:16:12.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:12.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:12.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:12.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:12.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:16:12.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:16:12.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:16:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:16:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:16:12.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:16:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:16:14.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:16:14.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:14.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:14.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:14.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:16:14.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:16:14.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:16:14.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:16:14.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:16:14.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:14.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:16:14.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:16:14.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:16:14.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:16:14.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:14.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:16:15.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:15.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:16:15.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:16:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:16:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:16:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:16:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:16:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:16:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:16:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:16:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:16:15.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:16:15.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:16:15.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:15.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:15.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:16:15.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:16:15.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:16:15.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:15.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:16:15.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:16:15.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:16:15.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:16:15.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:15.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:15.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:16:15.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:16:15.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:16:15.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:16:15.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:16:15.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:16:15.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:16:15.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:16:15.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:16:15.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:16:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:16:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:16:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:16:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:16:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:16:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:16:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:16:15.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:16:15.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:16:15.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:15.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:15.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:15.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:16:17.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 126 2026-02-20T21:16:17.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:16:17.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:16:17.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:17.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:16:17.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:16:17.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:17.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:17.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:16:17.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:17.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:16:17.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:16:17.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:16:17.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:17.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:16:17.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:16:17.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:17.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:17.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:17.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:16:17.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:16:17.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:16:17.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:16:17.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=126 2026-02-20T21:16:17.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:16:17.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:17.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:16:18.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:16:18.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:18.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:18.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:18.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:16:18.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:16:18.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:16:18.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:16:18.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=126 2026-02-20T21:16:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 126 = 126 2026-02-20T21:16:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-20T21:16:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:16:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:18.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:18.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:19.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-20T21:16:19.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:19.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:19.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:19.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:16:19.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:19.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:19.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:19.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:19.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:16:19.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:16:19.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:16:19.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:16:19.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:16:19.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:16:19.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:16 2026-02-20T21:16:19.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:16:19.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:16:19.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:16:19.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622170,"replay_state":"idle"} 2026-02-20T21:16:19.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:13 2026-02-20T21:16:19.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:16:19.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 125 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.e97b5f87-a934-4e78-9545-c9dbe173fa16 (peer_uuids:[]) 2026-02-20T21:16:19.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 126 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.6d2e9626-848c-4a02-a7a8-cd462d6726e0 (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:16:19.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:16:19.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:16:19.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:16:19.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:16:19.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:16:19.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:16:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:16:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:16:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:16:19.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:16:19.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:16:19.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:16:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:16' 2026-02-20T21:16:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-20 21:16:16' 2026-02-20T21:16:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:16:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:16:16' 2026-02-20T21:16:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:16:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:16:16cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622170,"replay_state":"idle"}2026-02-20 21:16:13' 2026-02-20T21:16:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:16:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:16:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:16' 2026-02-20T21:16:19.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:16:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:16:19.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:16:19.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:16:19.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:16:19.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:19.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:19.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:16:19.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:19.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:19.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:19.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:16:19.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:19.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:19.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:19.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:20.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:16:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:16:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:20.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:20.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:16:20.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:16:20.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:16:20.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:16:20.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622170,"replay_state":"idle"} 2026-02-20T21:16:20.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:16:20.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:13 2026-02-20T21:16:20.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:16:20.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:16:20.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:16:20.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:16:20.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:16' 2026-02-20T21:16:20.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:16:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:16:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:16:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622170,"replay_state":"idle"} 2026-02-20T21:16:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:16:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:16:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:16:20.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:16:20.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:16:20.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:16:20.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:16:20.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:16:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:13' 2026-02-20T21:16:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:16:13' 2026-02-20T21:16:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:16:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:16:13' 2026-02-20T21:16:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:20.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:20.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:16:20.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622170,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:16:13cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:16:16' 2026-02-20T21:16:20.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:16:20.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:16:20.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:13' 2026-02-20T21:16:20.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:16:20.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:16:20.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:16:20.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:16:20.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:16:20.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:20.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:20.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:16:20.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test 2026-02-20T21:16:20.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:20.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:20.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:20.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-20T21:16:21.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:16:21.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:16:21.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:21.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:21.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:21.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:16:21.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:21.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:21.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:21.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:16:21.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:16:21.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:21.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:16:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:16:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:16:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:16:22.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:16:22.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:16:22.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:22.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:16:22.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:16:22.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:16:22.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:16:22.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:22.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:16:22.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:22.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:16:22.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:22.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:16:22.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:16:22.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:22.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:16:22.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:22.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:16:22.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:16:22.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:16:22.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:16:22.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:16:22.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:22.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:16:22.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:16:22.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:16:22.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:22.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:22.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:16:22.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:16:22.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:16:22.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:22.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:16:22.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:16:22.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:16:22.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:16:22.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:22.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:22.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:16:22.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:16:22.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:16:22.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:16:22.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:22.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:22.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:22.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:22.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:16:23.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:16:23.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:16:23.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:16:23.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:16:23.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:16:23.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:23.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:16:23.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:23.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:23.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:23.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:23.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:23.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:23.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:23.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:16:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:16:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:16:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:16:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:16:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:16:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:16:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:23 2026-02-20T21:16:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:16:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:16:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:16:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:16:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:21' 2026-02-20T21:16:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:16:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:16:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:16:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:16:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:16:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:16:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:16:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:16:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:16:24.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:16:24.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:16:24.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:16:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:23' 2026-02-20T21:16:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:16:23' 2026-02-20T21:16:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:24.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:16:23' 2026-02-20T21:16:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:24.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:16:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:16:23cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:16:21' 2026-02-20T21:16:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:16:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:16:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:23' 2026-02-20T21:16:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:16:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:16:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:16:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:16:24.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:16:24.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:24.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:24.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:16:24.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:24.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:24.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:24.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:24.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:24.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:24.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:24.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:25.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:16:25.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:25.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:25.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:25.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:25.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:25.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:25.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:25.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:25.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:16:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:16:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:16:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:16:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:16:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:16:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:21 2026-02-20T21:16:25.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:16:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:16:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:16:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:16:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:23' 2026-02-20T21:16:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:16:25.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:16:25.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:16:25.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:16:25.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:16:25.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:16:25.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:16:25.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:16:25.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:16:25.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:16:25.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:16:25.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:16:25.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:21' 2026-02-20T21:16:25.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:16:21' 2026-02-20T21:16:25.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:25.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:25.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:25.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:25.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:16:21' 2026-02-20T21:16:25.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:25.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:25.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:16:25.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:16:21cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:16:23' 2026-02-20T21:16:25.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:16:25.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:16:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:21' 2026-02-20T21:16:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:16:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:16:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:16:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:16:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:16:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test 2026-02-20T21:16:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:16:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-20T21:16:26.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:16:26.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-20T21:16:26.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:26.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:26.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:26.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-20T21:16:26.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:26.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:26.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:26.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:16:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:16:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:27.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:16:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:16:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:16:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:16:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:16:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:16:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:16:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:16:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:16:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:16:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:16:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:16:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:16:27.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:27.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:16:27.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:16:27.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:27.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:16:27.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:16:27.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:16:27.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:16:27.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:16:27.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:16:27.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:16:27.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:16:27.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:16:27.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:27.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:16:27.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:16:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:16:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:16:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:16:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:16:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:16:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:16:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:16:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:16:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:16:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:16:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:16:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:16:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:16:27.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:16:27.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:16:27.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:16:27.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:16:27.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:16:27.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:16:27.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:16:28.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:28.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:16:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:16:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:16:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:16:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:16:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:16:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:16:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:16:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:16:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:16:28.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:16:28.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:28.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:16:28.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:16:28.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:16:28.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:16:28.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:16:28.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:16:28.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:16:28.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:16:28.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:16:28.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:16:28.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:28.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:16:28.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:16:28.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:16:28.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:16:28.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:16:28.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:16:28.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:16:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:16:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:16:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:16:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:16:28.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:16:28.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:16:28.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:16:28.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:16:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:16:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:16:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:16:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:16:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:16:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:16:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:16:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:16:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:16:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-20T21:16:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-20T21:16:31.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 129 2026-02-20T21:16:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:16:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:16:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:16:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:16:31.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:31.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:16:31.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:16:31.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:31.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:16:31.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:31.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:16:31.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:31.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:16:31.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:16:31.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:31.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:31.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:31.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:16:31.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:16:31.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:16:31.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:16:31.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=129 2026-02-20T21:16:31.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:16:31.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:31.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:16:32.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:16:32.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:32.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:32.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:32.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:16:32.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:16:32.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:16:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:16:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=129 2026-02-20T21:16:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 129 = 129 2026-02-20T21:16:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:16:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:16:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:33.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:16:33.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:33.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:33.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:33.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:16:33.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:33.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:33.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:33.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:33.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:16:33.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:16:33.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:16:33.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:16:33.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:16:33.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:16:33.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:33 2026-02-20T21:16:33.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:16:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:16:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:16:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622185,"replay_state":"idle"} 2026-02-20T21:16:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:26 2026-02-20T21:16:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:16:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 129 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.7f8c04c4-f49d-4005-b15f-e5600830b393 (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:16:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:16:33.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:16:33.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:16:33.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:16:33.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:16:33.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:16:33.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:16:33.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:16:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:16:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:16:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:16:33.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:16:33.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:33' 2026-02-20T21:16:33.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:16:33' 2026-02-20T21:16:33.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:33.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:33.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:33.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:16:33.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:16:33' 2026-02-20T21:16:33.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:33.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:33.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:16:33.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary44044409mirror.0trial1992026-02-20 21:16:33cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622185,"replay_state":"idle"}2026-02-20 21:16:26' 2026-02-20T21:16:33.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:16:33.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:16:33.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:33' 2026-02-20T21:16:33.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:16:33.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:16:33.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:16:33.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:16:33.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:16:33.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-20T21:16:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:16:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:34.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-20T21:16:34.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:16:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:16:34.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:16:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:16:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:16:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622185,"replay_state":"idle"} 2026-02-20T21:16:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:16:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:26 2026-02-20T21:16:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:16:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:16:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:16:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:16:34.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:33' 2026-02-20T21:16:34.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:16:34.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:16:34.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:16:34.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622185,"replay_state":"idle"} 2026-02-20T21:16:34.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:16:34.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:16:34.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:16:34.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:16:34.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:16:34.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:16:34.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:16:34.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:16:34.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:26' 2026-02-20T21:16:34.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-20 21:16:26' 2026-02-20T21:16:34.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:34.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:34.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:34.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:16:34.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:16:26' 2026-02-20T21:16:34.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:34.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:34.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:16:34.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622185,"replay_state":"idle"}45294534mirror.1trial1992026-02-20 21:16:26cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:16:33' 2026-02-20T21:16:34.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:16:34.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:16:34.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:26' 2026-02-20T21:16:34.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:16:34.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:16:34.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:16:34.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:16:34.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:16:34.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:34.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:34.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster1 mirror test 2026-02-20T21:16:34.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:34.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:34.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:34.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-20T21:16:34.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:16:34.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-20T21:16:34.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:34.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:34.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:34.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-20T21:16:34.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:34.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:34.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:34.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:16:34.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:16:34.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:34.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:16:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:35.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:16:35.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:16:35.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:16:35.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:16:35.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:16:35.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:16:35.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:35.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:16:35.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:16:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:16:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:16:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:16:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:16:35.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:35.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:16:35.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:16:35.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:35.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:16:35.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:16:35.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:16:35.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:16:35.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:16:35.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:16:35.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:16:35.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:16:35.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:16:35.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:35.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:16:35.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:35.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:16:35.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:16:35.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:16:35.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:16:35.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:16:35.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:16:35.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:16:35.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:16:35.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:35.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:35.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:16:35.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:16:35.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:16:35.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:16:35.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:16:35.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:16:35.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:16:35.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:16:35.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:16:35.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:16:36.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:16:36.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:16:36.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:16:36.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:36.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:16:36.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:16:36.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:36.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:16:36.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:16:36.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:16:36.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:16:36.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:16:36.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:16:36.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:16:36.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:16:36.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:16:36.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:36.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:16:36.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:16:36.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:16:36.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:16:36.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:16:36.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:16:36.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:16:36.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:16:36.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:16:36.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:16:36.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:36.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:16:36.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:16:36.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:16:36.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:16:36.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:16:36.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:16:36.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:16:36.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:16:36.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:16:36.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:16:36.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:16:36.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:16:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:16:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:16:36.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:16:36.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:16:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:16:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:16:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:16:38.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:16:38.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:38.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:38.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:38.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:16:38.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:16:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:16:38.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:16:38.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:16:38.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:16:38.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:38.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:16:38.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:16:38.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:16:38.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:38.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:16:38.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:16:38.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:16:38.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:38.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:16:38.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:16:38.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:38.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:16:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:16:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:16:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:16:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:16:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:16:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:16:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:16:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:16:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:16:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:16:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:16:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:16:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:16:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:16:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:16:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:16:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:16:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:16:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:16:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:16:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:16:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:16:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:16:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:16:38.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:16:38.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:16:38.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:16:38.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:16:38.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:16:38.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:16:38.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:38.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:16:38.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:16:38.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:38.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:16:38.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:16:38.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:16:38.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:16:38.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:16:38.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:16:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:16:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:16:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:16:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:16:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:16:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:16:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:16:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:16:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:16:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:16:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:16:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:16:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:16:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:16:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:16:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:16:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:16:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:16:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:16:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:16:38.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:16:38.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:16:38.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:16:38.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:16:38.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:16:38.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:16:38.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:16:38.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:16:38.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:16:38.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:38.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:16:38.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:38.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:38.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:38.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:38.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:38.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:38.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:38.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:16:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:39.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:39.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:39.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:16:39.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:16:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:16:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:16:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:16:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:16:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:38 2026-02-20T21:16:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:16:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:16:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:16:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:16:39.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:36' 2026-02-20T21:16:39.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:16:39.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:16:39.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:16:39.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:16:39.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:16:39.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:16:39.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:16:39.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:16:39.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:16:39.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:16:39.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:16:39.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:16:39.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:38' 2026-02-20T21:16:39.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:16:38' 2026-02-20T21:16:39.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:39.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:39.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:39.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:39.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:16:38' 2026-02-20T21:16:39.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:39.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:39.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:16:39.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:16:38cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:16:36' 2026-02-20T21:16:39.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:16:39.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:16:39.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:38' 2026-02-20T21:16:39.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:16:39.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:16:39.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:16:39.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:16:39.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:16:39.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:39.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:39.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:16:39.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:39.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:39.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:39.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:39.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:39.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:39.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:39.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:40.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:16:40.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:40.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:40.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:16:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:16:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:16:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:16:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:16:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:16:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:36 2026-02-20T21:16:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:16:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:16:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:16:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:16:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:38' 2026-02-20T21:16:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:16:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:16:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:16:40.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:16:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:16:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:16:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:16:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:16:40.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:16:40.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:16:40.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:16:40.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:16:40.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:36' 2026-02-20T21:16:40.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:16:36' 2026-02-20T21:16:40.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:40.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:16:36' 2026-02-20T21:16:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:16:40.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:16:36cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:16:38' 2026-02-20T21:16:40.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:16:40.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:16:40.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:36' 2026-02-20T21:16:40.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:16:40.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:16:40.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:16:40.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:16:40.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:16:40.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:40.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:40.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster2 mirror test 2026-02-20T21:16:40.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:40.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:40.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:40.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:16:40.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-20T21:16:41.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:16:41.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:16:41.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:41.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:41.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:41.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:16:41.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:41.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:41.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:41.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:16:41.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:16:41.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:41.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:42.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:16:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:16:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:16:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:16:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:16:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:16:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:16:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:16:42.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:16:42.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:16:42.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:42.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:16:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:16:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:16:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:16:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:16:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:16:42.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:16:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:16:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:16:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:16:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:16:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:16:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:16:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:16:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:16:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:16:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:16:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:16:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:16:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:16:42.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:42.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:42.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:16:42.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:16:42.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:16:42.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:16:42.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:16:42.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:16:42.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:16:42.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:16:42.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:16:42.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:16:42.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:42.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:16:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:16:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:16:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:16:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:16:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:16:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:16:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:16:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:16:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:16:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:16:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:16:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:44.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:16:44.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:44.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:16:44.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:16:44.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:44.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:16:44.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:44.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:16:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:16:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:16:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:16:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:16:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:16:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:16:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:16:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:16:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:16:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:16:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:16:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:16:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:16:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:16:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:16:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:16:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:16:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:16:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:16:45.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:16:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:16:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:16:45.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:16:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:16:45.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:16:45.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:16:45.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:45.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:16:45.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:16:45.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:45.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:16:45.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:16:45.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:45.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:16:45.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:16:45.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:16:45.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:45.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:45.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:45.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:16:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 132 2026-02-20T21:16:48.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:16:48.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:16:48.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:48.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:16:48.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:16:48.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:48.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:48.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:16:48.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:48.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:16:48.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:16:48.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:16:48.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:48.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:16:48.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:16:48.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:48.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:48.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:48.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:16:48.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:16:48.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:16:48.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:16:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=132 2026-02-20T21:16:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:16:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:16:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:16:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:48.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:16:48.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:16:48.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:16:48.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:16:48.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=132 2026-02-20T21:16:48.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 132 = 132 2026-02-20T21:16:48.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:48.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-20T21:16:48.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:48.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:48.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:48.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:16:48.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:48.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:48.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:48.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-20T21:16:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:49.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:49.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:16:49.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:49.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:49.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:49.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:49.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:16:49.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:16:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:16:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:16:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:16:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:16:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:46 2026-02-20T21:16:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:16:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:16:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:16:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622200,"replay_state":"idle"} 2026-02-20T21:16:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:43 2026-02-20T21:16:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:16:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 132 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.134d73b7-ab37-468b-a308-bb0244db72b2 (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:16:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:16:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:16:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:16:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:16:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:16:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:16:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:16:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:16:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:16:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:16:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:16:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:16:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:46' 2026-02-20T21:16:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-20 21:16:46' 2026-02-20T21:16:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:16:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:16:46' 2026-02-20T21:16:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:16:49.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:16:46cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622200,"replay_state":"idle"}2026-02-20 21:16:43' 2026-02-20T21:16:49.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:16:49.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:16:49.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:46' 2026-02-20T21:16:49.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:16:49.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:16:49.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:16:49.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:16:49.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:16:49.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:49.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:49.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:16:49.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:49.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:49.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:49.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:16:49.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:49.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:49.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:49.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:50.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:16:50.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:50.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:50.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:50.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:16:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:16:50.500 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:16:50.500 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:16:50.500 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:16:50.500 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622200,"replay_state":"idle"} 2026-02-20T21:16:50.500 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:16:50.500 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:43 2026-02-20T21:16:50.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:16:50.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:16:50.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:16:50.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:16:50.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:46' 2026-02-20T21:16:50.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:16:50.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:16:50.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:16:50.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622200,"replay_state":"idle"} 2026-02-20T21:16:50.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:16:50.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:16:50.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:16:50.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:16:50.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:16:50.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:16:50.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:16:50.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:16:50.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:43' 2026-02-20T21:16:50.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:16:43' 2026-02-20T21:16:50.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:50.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:50.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:50.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:16:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:16:43' 2026-02-20T21:16:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:50.507 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:16:50.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622200,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:16:43cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:16:46' 2026-02-20T21:16:50.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:16:50.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:16:50.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:43' 2026-02-20T21:16:50.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:16:50.541 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:16:50.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:16:50.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:16:50.543 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:16:50.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:50.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:50.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:16:50.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test 2026-02-20T21:16:50.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:50.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:50.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:50.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-20T21:16:51.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:16:51.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:16:51.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:51.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:51.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:51.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:16:51.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:51.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:51.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:51.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:16:51.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:16:51.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:51.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:52.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:16:52.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:52.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:52.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:52.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:16:52.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:16:52.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:16:52.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:16:52.499 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:16:52.499 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:16:52.499 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:52.499 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:16:52.499 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:16:52.499 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:16:52.499 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:52.499 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:16:52.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:52.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:16:52.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:52.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:16:52.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:16:52.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:52.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:16:52.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:52.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:16:52.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:16:52.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:16:52.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:16:52.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:16:52.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:52.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:16:52.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:16:52.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:16:52.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:52.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:52.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:16:52.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:16:52.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:16:52.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:52.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:16:52.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:16:52.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:16:52.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:16:52.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:52.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:52.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:16:52.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:16:52.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:16:52.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:16:52.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:52.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:52.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:52.514 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:52.514 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:16:52.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:16:52.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:16:52.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:16:52.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:16:52.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:16:52.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:16:52.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:16:52.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:52.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:16:54.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:16:54.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:54.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:54.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:54.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:16:54.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:16:54.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:16:54.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:16:54.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:16:54.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:54.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:16:54.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:16:54.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:16:54.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:54.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:16:54.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:16:54.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:54.569 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:16:54.569 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:54.569 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:16:54.569 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:16:54.569 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:54.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:16:54.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:54.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:16:54.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:16:54.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:16:54.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:16:54.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:16:54.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:54.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:16:54.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:16:54.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:16:54.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:54.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:54.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:16:54.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:16:54.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:16:54.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:54.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:16:54.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:16:54.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:16:54.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:16:54.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:54.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:54.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:16:54.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:16:54.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:16:54.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:16:54.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:54.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:54.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:54.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:54.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:16:54.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:16:54.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:16:54.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:16:54.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:16:54.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:16:54.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:54.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:16:54.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:54.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:54.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:54.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:54.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:54.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:54.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:55.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:16:55.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:55.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:55.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:55.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:55.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:55.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:55.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:55.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:55.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:16:55.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:16:55.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:16:55.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:16:55.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:16:55.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:16:55.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:53 2026-02-20T21:16:55.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:16:55.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:16:55.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:16:55.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:16:55.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:51' 2026-02-20T21:16:55.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:16:55.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:16:55.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:16:55.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:16:55.666 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:16:55.666 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:16:55.666 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:16:55.666 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:16:55.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:16:55.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:16:55.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:16:55.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:16:55.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:53' 2026-02-20T21:16:55.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:16:53' 2026-02-20T21:16:55.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:55.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:55.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:55.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:55.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:16:53' 2026-02-20T21:16:55.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:55.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:55.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:16:55.702 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:16:53cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:16:51' 2026-02-20T21:16:55.702 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:16:55.702 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:16:55.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:53' 2026-02-20T21:16:55.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:16:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:16:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:16:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:16:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:16:55.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:55.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:55.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:16:55.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:55.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:55.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:55.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:55.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:55.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:55.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:55.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:16:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:56.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:56.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:16:56.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:16:56.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:56.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:56.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:16:56.739 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:16:56.739 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:16:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:16:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:16:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:16:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:51 2026-02-20T21:16:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:16:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:16:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:16:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:16:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:16:53' 2026-02-20T21:16:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:16:56.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:16:56.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:16:56.742 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:16:56.742 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:16:56.742 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:16:56.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:16:56.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:16:56.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:16:56.744 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:16:56.744 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:16:56.744 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:16:56.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:51' 2026-02-20T21:16:56.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:16:51' 2026-02-20T21:16:56.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:56.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:56.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:56.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:16:56.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:16:51' 2026-02-20T21:16:56.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:16:56.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:16:56.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:16:56.775 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:16:51cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:16:53' 2026-02-20T21:16:56.775 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:16:56.776 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:16:56.776 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:16:51' 2026-02-20T21:16:56.777 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:16:56.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:16:56.779 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:16:56.779 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:16:56.779 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:16:56.780 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:56.780 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:16:56.780 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test 2026-02-20T21:16:56.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:16:56.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:56.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:56.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:16:56.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-20T21:16:57.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:16:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-20T21:16:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-20T21:16:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:16:57.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:16:57.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:57.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:16:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:16:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:16:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:16:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:16:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:16:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:16:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:16:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:16:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:16:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:16:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:16:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:16:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:16:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:16:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:16:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:16:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:16:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:16:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:16:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:16:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:16:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:16:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:16:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:16:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:16:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:16:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:16:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:16:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:16:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:16:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:16:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:16:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:16:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:16:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:16:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:16:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:16:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:16:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:16:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:16:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:16:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:16:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:16:58.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:16:58.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:16:58.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:16:58.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:16:59.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:16:59.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:16:59.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:16:59.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:59.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:16:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:16:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:16:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:16:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:16:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:16:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:16:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:16:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:16:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:16:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:16:59.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:16:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:16:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:16:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:16:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:16:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:16:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:16:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:16:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:16:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:16:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:16:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:16:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:16:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:16:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:16:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:16:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:16:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:16:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:16:59.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:16:59.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:16:59.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:16:59.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:16:59.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:16:59.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:16:59.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:16:59.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:16:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:16:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:17:01.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:17:01.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:01.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:01.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:01.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:17:01.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:17:01.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:17:01.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:17:01.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:17:01.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:17:01.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:01.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:17:01.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:17:01.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:17:01.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:01.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:17:01.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:17:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:17:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:17:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:17:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:17:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:17:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:17:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:17:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:17:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:17:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:17:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:17:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:17:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:17:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:17:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:17:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:17:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:17:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:17:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:17:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:17:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:17:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:17:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:17:01.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:17:01.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:01.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:01.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:01.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:17:01.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:01.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:17:01.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:17:01.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:01.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:17:01.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:17:01.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:01.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:17:01.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:17:01.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:17:01.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:01.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:17:01.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:17:01.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:17:01.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:17:01.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:17:01.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:01.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:17:01.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:17:01.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:17:01.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:17:01.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:17:01.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:17:01.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:17:01.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:17:01.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:01.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:01.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:01.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:17:01.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:17:01.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:17:01.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:17:01.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:17:01.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:17:01.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:17:01.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:01.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:01.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:01.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:17:01.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:17:01.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:17:01.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:17:01.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:17:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:17:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:17:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:17:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:05.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:05.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:17:05.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:17:05.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:17:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:17:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:17:05.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:17:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:17:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:17:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:17:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:17:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:17:05.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:17:05.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:05.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:17:05.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:17:05.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:05.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:17:05.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:17:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:17:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:17:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:17:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:17:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:17:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:17:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:17:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:17:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:17:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:17:05.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:17:05.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:17:05.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:17:05.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:05.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:05.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:05.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:05.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:17:05.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:17:05.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:17:05.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:17:05.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:17:05.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:17:05.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:05.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:05.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:05.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:17:05.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:17:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:17:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:17:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:17:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:17:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:17:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:17:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:17:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:17:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:17:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:17:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:17:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:17:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:17:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:17:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:17:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:17:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:17:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:17:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:17:05.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:05.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:05.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:05.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:17:05.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:17:05.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:17:05.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:17:05.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:17:05.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:17:05.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:17:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:05.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:05.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:17:05.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:17:05.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:17:05.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:17:05.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:17:05.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:17:05.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:17:05.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:17:05.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:05.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:17:05.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:17:05.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:05.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:17:05.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:17:05.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:05.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:17:05.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:17:05.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-20T21:17:05.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:05.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:05.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:05.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-20T21:17:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 135 2026-02-20T21:17:08.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:17:08.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:17:08.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:08.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:17:08.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:17:08.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:08.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:17:08.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:17:08.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:08.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:17:08.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:08.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:17:08.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:17:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:17:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:17:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:17:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:17:08.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:17:08.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=135 2026-02-20T21:17:08.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:17:08.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:08.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:17:09.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:17:09.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:09.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:17:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:17:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:17:09.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:17:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=135 2026-02-20T21:17:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 135 = 135 2026-02-20T21:17:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:17:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:17:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:17:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:17:10.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:10.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:10.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:10.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:17:10.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:10.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:10.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:10.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:10.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:17:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:17:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:17:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:17:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:17:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:17:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:08 2026-02-20T21:17:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:17:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:17:10.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:17:10.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622216,"replay_state":"idle"} 2026-02-20T21:17:10.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:01 2026-02-20T21:17:10.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:17:10.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 135 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.eeb000d5-acb3-4a0e-8621-c4531782e5e5 (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:17:10.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:17:10.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:17:10.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:17:10.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:17:10.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:17:10.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:17:10.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:17:10.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:17:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:17:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:17:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:17:10.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:17:10.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:08' 2026-02-20T21:17:10.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:17:08' 2026-02-20T21:17:10.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:10.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:10.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:10.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:17:10.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:17:08' 2026-02-20T21:17:10.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:10.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:10.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:17:10.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary44044409mirror.0trial1992026-02-20 21:17:08cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622216,"replay_state":"idle"}2026-02-20 21:17:01' 2026-02-20T21:17:10.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:17:10.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:17:10.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:08' 2026-02-20T21:17:10.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:17:10.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:17:10.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:17:10.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:17:10.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:17:10.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:10.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:10.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-20T21:17:10.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:10.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:10.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:10.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:17:10.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:10.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:10.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:10.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:17:11.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-20T21:17:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:17:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:17:11.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:17:11.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:17:11.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:17:11.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622216,"replay_state":"idle"} 2026-02-20T21:17:11.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:17:11.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:01 2026-02-20T21:17:11.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:17:11.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:17:11.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:17:11.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:17:11.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:08' 2026-02-20T21:17:11.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:17:11.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:17:11.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:17:11.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622216,"replay_state":"idle"} 2026-02-20T21:17:11.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:17:11.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:17:11.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:17:11.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:17:11.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:17:11.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:17:11.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:17:11.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:17:11.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:01' 2026-02-20T21:17:11.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-20 21:17:01' 2026-02-20T21:17:11.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:17:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:17:01' 2026-02-20T21:17:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:17:11.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622216,"replay_state":"idle"}45294534mirror.1trial1992026-02-20 21:17:01cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:17:08' 2026-02-20T21:17:11.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:17:11.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:17:11.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:01' 2026-02-20T21:17:11.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:17:11.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:17:11.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:17:11.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:17:11.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:17:11.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:11.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:11.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster1 mirror test 2026-02-20T21:17:11.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:11.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:11.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:11.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-20T21:17:11.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:17:11.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-20T21:17:11.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:11.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:11.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:11.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-20T21:17:11.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:11.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:11.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:11.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:17:11.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:17:11.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:11.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:17:12.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:17:12.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:12.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:12.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:12.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:17:12.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:17:12.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:17:12.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:17:12.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:17:12.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:12.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:17:12.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:17:12.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:17:12.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:17:12.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:12.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:17:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:17:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:17:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:17:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:17:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:17:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:17:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:17:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:17:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:17:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:17:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:17:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:17:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:17:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:17:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:17:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:17:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:17:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:17:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:17:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:17:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:17:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:17:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:17:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:17:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:17:13.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:13.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:13.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:13.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:17:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:17:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:17:13.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:13.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:17:13.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:17:13.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:13.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:17:13.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:17:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:17:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:17:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:17:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:17:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:17:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:17:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:17:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:17:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:17:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:17:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:17:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:17:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:17:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:17:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:17:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:17:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:17:13.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:17:13.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:17:13.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:17:13.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:17:13.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:13.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:13.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:13.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:17:13.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:17:13.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:17:13.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:17:13.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:17:13.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:17:13.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:17:13.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:17:13.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:13.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:17:15.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:17:15.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:15.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:15.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:15.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:17:15.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:17:15.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:17:15.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:17:15.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:17:15.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:17:15.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:15.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:17:15.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:17:15.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:17:15.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:15.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:17:15.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:17:15.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:17:15.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:15.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:17:15.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:17:15.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:15.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:17:15.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:17:15.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:17:15.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:15.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:17:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:17:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:17:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:17:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:17:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:17:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:17:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:17:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:17:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:17:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:17:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:17:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:17:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:17:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:17:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:17:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:17:15.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:17:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:17:15.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:15.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:17:15.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:17:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:17:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:17:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:17:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:17:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:17:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:17:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:17:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:17:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:17:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:17:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:17:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:17:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:17:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:17:15.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:17:15.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:17:15.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:17:15.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:17:15.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:15.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:15.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:15.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:17:15.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:17:15.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:17:15.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:17:15.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:17:15.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:17:15.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:17:15.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:15.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:15.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:15.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:17:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:17:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:17:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:17:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:17:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:17:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:17:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:15.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:17:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:17:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:17:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:17:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:17:16.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:17:16.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:17:16.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:17:16.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:17:16.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:17:16.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:13 2026-02-20T21:17:16.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:17:16.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:17:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:17:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:17:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:13' 2026-02-20T21:17:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:17:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:17:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:17:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:17:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:17:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:17:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:17:16.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:17:16.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:17:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:17:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:17:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:17:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:13' 2026-02-20T21:17:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:17:13' 2026-02-20T21:17:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:17:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:17:13' 2026-02-20T21:17:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:17:16.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:17:13cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image demoted2026-02-20 21:17:13' 2026-02-20T21:17:16.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:17:16.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:17:16.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:13' 2026-02-20T21:17:16.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:17:16.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:17:16.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:17:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:17:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:17:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:17:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:17:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:17:17.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:17:17.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:17.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:17.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:17.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:17:17.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:17.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:17.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:17.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:17.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:17:17.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:17:17.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:17:17.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:17:17.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:17:17.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:17:17.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:16 2026-02-20T21:17:17.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:17:17.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:17:17.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:17:17.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:17:17.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:13' 2026-02-20T21:17:17.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:17:17.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:17:17.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:17:17.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:17:17.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:17:17.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:17:17.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:17:17.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:17:17.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:17:17.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:17:17.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:17:17.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:17:17.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:16' 2026-02-20T21:17:17.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:17:16' 2026-02-20T21:17:17.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:17.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:17.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:17.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:17:17.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:17:16' 2026-02-20T21:17:17.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:17.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:17.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:17:17.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:17:16cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:17:13' 2026-02-20T21:17:17.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:17:17.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:17:17.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:16' 2026-02-20T21:17:17.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:17:17.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:17:17.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:17:17.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:17:17.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:17:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster2 mirror test 2026-02-20T21:17:17.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:17.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:17.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:17.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:17:17.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-20T21:17:18.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:17:18.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:17:18.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:18.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:18.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:18.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:17:18.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:18.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:18.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:18.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:17:18.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:17:18.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:18.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:17:19.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:17:19.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:19.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:19.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:19.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:17:19.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:17:19.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:17:19.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:17:19.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:17:19.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:17:19.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:19.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:17:19.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:17:19.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:17:19.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:19.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:17:19.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:19.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:17:19.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:19.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:17:19.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:17:19.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:19.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:17:19.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:19.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:17:19.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:19.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:17:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:17:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:17:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:17:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:17:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:17:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:17:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:17:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:17:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:17:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:17:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:17:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:17:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:17:19.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:17:19.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:19.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:19.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:19.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:17:19.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:17:19.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:17:19.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:17:19.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:17:19.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:17:19.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:17:19.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:17:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:17:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:17:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:17:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:17:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:17:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:17:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:17:19.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:19.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:19.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:19.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:17:22.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 138 2026-02-20T21:17:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:17:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:17:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:17:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:17:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:22.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:17:22.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:17:22.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:17:22.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:17:22.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:17:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:17:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:17:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:17:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:17:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:17:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=138 2026-02-20T21:17:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:17:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:17:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:17:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:17:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:17:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:17:23.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:17:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=138 2026-02-20T21:17:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 138 = 138 2026-02-20T21:17:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-20T21:17:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:23.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:17:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:17:24.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-20T21:17:24.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:24.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:24.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:24.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:17:24.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:24.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:24.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:24.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:24.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:17:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:17:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:17:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:17:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:17:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:17:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:21 2026-02-20T21:17:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:17:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:17:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:17:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622237,"replay_state":"idle"} 2026-02-20T21:17:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:18 2026-02-20T21:17:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:17:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 138 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.6e589d30-5eb6-4c26-96ab-c756daada2c3 (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:17:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:17:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:17:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:17:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:17:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:17:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:17:24.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:17:24.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:17:24.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:17:24.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:17:24.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:17:24.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:17:24.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:21' 2026-02-20T21:17:24.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-20 21:17:21' 2026-02-20T21:17:24.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:24.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:24.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:24.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:17:24.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:17:21' 2026-02-20T21:17:24.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:24.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:24.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:17:24.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:17:21cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622237,"replay_state":"idle"}2026-02-20 21:17:18' 2026-02-20T21:17:24.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:17:24.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:17:24.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:21' 2026-02-20T21:17:24.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:17:24.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:17:24.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:17:24.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:17:24.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:17:24.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:24.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:24.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:17:24.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:24.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:24.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:24.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:17:24.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:24.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:24.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:24.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:17:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:17:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:17:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:25.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:17:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:17:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:17:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:17:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622237,"replay_state":"idle"} 2026-02-20T21:17:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:17:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:18 2026-02-20T21:17:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:17:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:17:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:17:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:17:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:21' 2026-02-20T21:17:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:17:25.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:17:25.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:17:25.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622237,"replay_state":"idle"} 2026-02-20T21:17:25.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:17:25.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:17:25.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:17:25.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:17:25.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:17:25.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:17:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:17:25.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:17:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:18' 2026-02-20T21:17:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:17:18' 2026-02-20T21:17:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:17:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:17:18' 2026-02-20T21:17:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:25.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:25.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:17:25.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622237,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:17:18cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:17:21' 2026-02-20T21:17:25.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:17:25.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:17:25.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:18' 2026-02-20T21:17:25.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:17:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:17:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:17:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:17:25.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:17:25.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:25.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:25.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:17:25.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test 2026-02-20T21:17:25.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:25.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:25.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:25.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-20T21:17:26.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:17:26.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:17:26.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:26.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:26.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:26.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:17:26.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:26.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:26.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:26.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:17:26.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:17:26.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:26.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:17:27.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:17:27.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:27.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:27.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:27.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:17:27.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:17:27.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:17:27.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:17:27.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:17:27.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:17:27.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:27.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:17:27.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:17:27.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:17:27.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:27.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:17:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:17:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:17:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:17:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:17:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:17:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:27.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:17:27.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:17:27.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:17:27.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:27.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:17:27.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:17:27.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:17:27.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:27.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:27.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:17:27.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:17:27.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:17:27.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:27.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:17:27.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:27.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:27.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:17:27.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:27.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:27.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:17:27.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:17:27.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:17:27.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:17:27.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:27.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:27.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:27.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:27.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:17:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:17:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:17:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:17:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:17:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:17:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:17:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:17:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:17:29.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:17:29.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:29.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:29.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:17:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:17:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:17:29.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:17:29.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:17:29.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:17:29.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:29.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:17:29.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:17:29.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:17:29.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:29.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:17:29.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:29.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:17:29.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:29.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:17:29.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:17:29.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:29.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:17:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:17:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:17:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:17:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:17:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:17:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:17:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:17:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:29.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:17:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:17:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:17:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:17:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:17:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:17:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:17:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:17:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:17:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:17:29.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:17:29.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:17:29.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:17:29.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:17:29.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:17:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:17:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:17:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:17:30.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:17:30.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:30.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:30.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:30.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:17:30.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:30.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:30.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:30.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:30.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:17:30.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:17:30.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:17:30.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:17:30.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:17:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:17:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:28 2026-02-20T21:17:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:17:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:17:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:17:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:17:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:26' 2026-02-20T21:17:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:17:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:17:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:17:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:17:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:17:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:17:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:17:30.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:17:30.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:17:30.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:17:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:17:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:17:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:28' 2026-02-20T21:17:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:17:28' 2026-02-20T21:17:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:17:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:17:28' 2026-02-20T21:17:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:17:30.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:17:28cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:17:26' 2026-02-20T21:17:30.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:17:30.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:17:30.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:28' 2026-02-20T21:17:30.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:17:30.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:17:30.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:17:30.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:17:30.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:17:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:17:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:17:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:30.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:17:31.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:17:31.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:31.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:31.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:31.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:17:31.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:31.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:31.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:31.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:31.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:17:31.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:17:31.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:17:31.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:17:31.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:17:31.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:17:31.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:26 2026-02-20T21:17:31.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:17:31.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:17:31.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:17:31.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:17:31.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:28' 2026-02-20T21:17:31.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:17:31.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:17:31.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:17:31.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:17:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:17:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:17:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:17:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:17:31.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:17:31.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:17:31.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:17:31.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:17:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:26' 2026-02-20T21:17:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:17:26' 2026-02-20T21:17:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:17:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:17:26' 2026-02-20T21:17:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:17:31.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:17:26cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:17:28' 2026-02-20T21:17:31.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:17:31.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:17:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:26' 2026-02-20T21:17:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:17:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:17:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:17:31.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:17:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test 2026-02-20T21:17:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:17:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-20T21:17:33.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:17:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-20T21:17:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-20T21:17:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:17:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:17:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:17:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:17:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:17:34.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:17:34.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:17:34.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:17:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:17:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:17:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:17:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:17:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:17:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:17:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:17:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:17:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:17:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:17:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:17:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:17:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:17:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:17:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:17:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:17:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:17:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:17:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:17:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:17:34.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:17:34.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:17:34.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:17:34.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:17:34.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:17:34.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:34.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:34.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:34.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:34.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:17:34.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:17:34.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:17:34.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:17:34.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:17:34.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:17:34.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:34.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:34.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:34.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:17:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:17:34.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:17:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:17:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:17:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:17:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:17:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:17:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:17:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:17:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:17:34.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:17:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:17:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:17:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:17:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:17:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:17:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:17:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:17:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:17:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:17:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:17:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:17:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:17:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:17:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:17:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:17:34.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:17:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:17:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:17:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:17:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:17:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:17:34.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:17:34.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:34.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:17:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:17:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:17:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:17:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:17:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:17:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:17:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:17:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:17:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:17:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:17:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:17:36.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:17:36.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:17:36.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:36.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:17:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:17:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:17:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:17:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:17:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:17:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:17:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:17:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:17:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:17:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:17:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:17:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:17:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:17:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:17:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:17:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:17:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:17:36.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:17:36.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:17:36.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:17:36.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:17:36.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:17:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:17:36.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:36.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:17:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:17:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:17:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:17:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:17:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:17:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:17:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:17:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:17:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:17:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:17:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:17:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:17:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:17:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:17:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:17:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:17:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:17:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:17:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:17:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:17:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:17:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:17:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:17:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:17:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:17:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:17:36.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:36.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:36.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:36.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:17:36.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:17:36.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:17:36.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:17:36.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:17:36.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:17:36.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:36.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:17:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:17:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:17:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:17:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:17:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:17:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:17:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:17:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:17:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:17:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:40.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:17:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:17:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:17:40.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:17:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:17:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:17:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:17:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:17:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:17:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:17:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:17:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:17:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:17:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:17:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:17:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:17:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:17:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:17:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:17:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:17:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:17:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:17:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:17:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:17:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:17:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:17:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:17:40.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:40.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:40.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:40.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:17:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:17:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:17:40.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:17:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:17:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:17:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:17:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:17:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:17:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:17:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:17:40.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:17:40.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:17:40.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:40.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:17:40.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:17:40.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:17:40.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:17:40.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:17:40.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:17:40.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:17:40.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:17:40.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:40.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:40.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:40.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:17:40.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:17:40.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:17:40.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:17:40.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:17:40.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:17:40.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:17:40.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:40.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:40.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:40.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:17:40.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:17:40.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:17:40.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:17:40.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:17:40.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:17:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:17:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:17:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:17:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:17:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:17:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:17:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:17:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:17:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-20T21:17:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-20T21:17:42.912 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 141 2026-02-20T21:17:42.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:17:42.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:17:42.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:42.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:17:42.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:17:42.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:42.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:17:42.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:17:42.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:42.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:17:42.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:17:42.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:42.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:17:42.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:17:42.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:42.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:42.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:42.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:17:42.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:17:42.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:17:42.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:17:42.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=141 2026-02-20T21:17:42.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:17:42.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:42.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:17:43.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:17:43.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:43.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:17:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:17:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:17:43.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:17:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=141 2026-02-20T21:17:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 141 = 141 2026-02-20T21:17:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:43.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:17:43.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:43.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:43.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:43.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:17:43.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:43.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:43.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:43.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:17:44.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:17:44.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:44.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:44.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:44.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:17:44.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:44.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:44.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:44.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:44.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:17:44.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:17:44.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:17:44.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:17:44.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:17:44.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:17:44.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:43 2026-02-20T21:17:44.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:17:44.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:17:44.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:17:44.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622251,"replay_state":"idle"} 2026-02-20T21:17:44.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:36 2026-02-20T21:17:44.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:17:44.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 140 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.3f2afbfd-7d49-4afd-bce7-27d3ae01fe17 (peer_uuids:[]) 2026-02-20T21:17:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 141 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.d1206a17-a888-4003-b0a4-24d35607d167 (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:17:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:17:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:17:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:17:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:17:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:17:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:17:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:17:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:17:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:17:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:17:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:17:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:17:44.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:43' 2026-02-20T21:17:44.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:17:43' 2026-02-20T21:17:44.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:44.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:44.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:44.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:17:44.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:17:43' 2026-02-20T21:17:44.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:44.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:44.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:17:44.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary44044409mirror.0trial1992026-02-20 21:17:43cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622251,"replay_state":"idle"}2026-02-20 21:17:36' 2026-02-20T21:17:44.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:17:44.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:17:44.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:43' 2026-02-20T21:17:44.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:17:44.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:17:44.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:17:44.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:17:44.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:17:44.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:44.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:44.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-20T21:17:44.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:44.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:44.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:44.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:17:44.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:44.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:44.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:44.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:17:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-20T21:17:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:17:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:17:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:17:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:17:45.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:17:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622251,"replay_state":"idle"} 2026-02-20T21:17:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:17:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:36 2026-02-20T21:17:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:17:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:17:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:17:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:17:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:43' 2026-02-20T21:17:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:17:45.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:17:45.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:17:45.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622251,"replay_state":"idle"} 2026-02-20T21:17:45.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:17:45.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:17:45.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:17:45.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:17:45.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:17:45.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:17:45.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:17:45.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:17:45.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:36' 2026-02-20T21:17:45.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-20 21:17:36' 2026-02-20T21:17:45.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:45.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:45.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:45.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:17:45.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:17:36' 2026-02-20T21:17:45.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:45.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:45.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:17:45.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622251,"replay_state":"idle"}45294534mirror.1trial1992026-02-20 21:17:36cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:17:43' 2026-02-20T21:17:45.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:17:45.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:17:45.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:36' 2026-02-20T21:17:45.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:17:45.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:17:45.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:17:45.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:17:45.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:17:45.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:45.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster1 mirror test 2026-02-20T21:17:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:45.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-20T21:17:46.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:17:46.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-20T21:17:46.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:46.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:46.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:46.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-20T21:17:46.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:46.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:46.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:46.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:17:46.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:17:46.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:46.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:17:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:17:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:47.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:17:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:17:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:17:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:17:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:17:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:17:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:17:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:17:47.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:17:47.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:47.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:17:47.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:17:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:17:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:17:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:17:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:17:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:17:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:17:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:17:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:17:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:17:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:17:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:17:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:17:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:17:47.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:17:47.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:17:47.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:17:47.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:17:47.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:17:47.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:47.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:47.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:47.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:47.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:17:47.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:17:47.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:17:47.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:17:47.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:17:47.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:17:47.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:47.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:47.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:17:47.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:17:47.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:47.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:17:47.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:17:47.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:17:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:17:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:17:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:17:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:17:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:17:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:17:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:17:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:17:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:17:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:17:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:17:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:17:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:17:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:17:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:17:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:17:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:17:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:17:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:17:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:17:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:17:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:17:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:17:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:17:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:17:48.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:48.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:17:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:17:48.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:17:48.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:17:48.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:17:48.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:17:48.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:17:48.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:48.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:17:48.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:48.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:48.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:48.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:17:48.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:48.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:17:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:17:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:17:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:17:49.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:17:49.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:17:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:17:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:17:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:17:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:48 2026-02-20T21:17:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:17:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:17:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:17:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:17:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:47' 2026-02-20T21:17:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:17:49.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:17:49.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:17:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:17:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:17:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:17:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:17:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:17:49.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:17:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:17:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:17:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:17:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:48' 2026-02-20T21:17:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:17:48' 2026-02-20T21:17:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:17:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:17:48' 2026-02-20T21:17:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:49.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:17:49.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:17:48cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image demoted2026-02-20 21:17:47' 2026-02-20T21:17:49.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:17:49.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:17:49.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:48' 2026-02-20T21:17:49.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:17:49.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:17:49.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:17:49.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:17:49.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:17:49.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:49.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:49.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:17:49.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:49.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:49.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:49.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:17:49.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:49.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:49.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:49.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:17:50.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:17:50.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:50.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:50.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:50.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:17:50.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:50.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:50.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:50.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:50.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:17:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:17:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:17:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:17:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:17:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:17:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:47 2026-02-20T21:17:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:17:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:17:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:17:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:17:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:48' 2026-02-20T21:17:50.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:17:50.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:17:50.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:17:50.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image demoted 2026-02-20T21:17:50.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:17:50.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:17:50.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:17:50.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:17:50.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:17:50.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:17:50.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:17:50.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:17:50.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:47' 2026-02-20T21:17:50.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:17:47' 2026-02-20T21:17:50.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:50.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:50.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:50.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:17:50.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:17:47' 2026-02-20T21:17:50.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:50.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:50.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:17:50.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image demoted45294534mirror.1trial1992026-02-20 21:17:47cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:17:48' 2026-02-20T21:17:50.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:17:50.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:17:50.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:47' 2026-02-20T21:17:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:17:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:17:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:17:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:17:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:17:50.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:50.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:50.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster2 mirror test 2026-02-20T21:17:50.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:50.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:50.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:50.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:17:50.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-20T21:17:51.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:17:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:17:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:17:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:17:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:17:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:51.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:17:52.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:17:52.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:52.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:52.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:52.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:17:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:17:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:17:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:17:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:17:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:17:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:17:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:17:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:17:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:17:52.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:52.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:17:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:17:53.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:17:53.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:17:53.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:17:53.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:17:53.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:17:53.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:53.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:17:55.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:17:55.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:55.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:55.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:55.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:17:55.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:17:55.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:17:55.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:17:55.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:17:55.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:17:55.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:55.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:17:55.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:17:55.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:17:55.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:55.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:17:55.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:55.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:17:55.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:55.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:17:55.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:17:55.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:17:55.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:17:55.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:55.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:17:55.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:55.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:17:55.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:17:55.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:17:55.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:55.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:17:55.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:17:55.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:17:55.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:55.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:55.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:17:55.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:17:55.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:17:55.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:55.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:17:55.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:55.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:55.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:17:55.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:17:55.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:55.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:17:55.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:17:55.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:17:55.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:17:55.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:55.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:55.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:55.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:55.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:17:55.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:17:55.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:17:55.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:17:55.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:17:55.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:17:55.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:17:55.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:17:55.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:55.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:17:55.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:17:55.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:55.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:17:55.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:17:55.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:55.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:17:55.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:17:55.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:17:55.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:55.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:55.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:55.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:17:57.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 144 2026-02-20T21:17:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:17:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:17:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:17:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:17:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:17:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:17:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:17:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:17:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:17:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:17:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:17:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:17:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:17:57.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:17:57.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=144 2026-02-20T21:17:57.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:17:57.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:57.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:17:58.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:17:58.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:58.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:58.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:58.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:17:58.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:17:58.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:17:58.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:17:58.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=144 2026-02-20T21:17:58.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 144 = 144 2026-02-20T21:17:58.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:58.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-20T21:17:58.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:58.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:58.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:58.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:17:58.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:58.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:58.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:58.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:17:59.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-20T21:17:59.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:59.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:59.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:59.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:17:59.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:59.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:59.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:59.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:59.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:17:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:17:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:17:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:17:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:17:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:17:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:56 2026-02-20T21:17:59.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:17:59.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:17:59.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:17:59.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622275,"remote_snapshot_timestamp":1771622275,"replay_state":"idle"} 2026-02-20T21:17:59.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:58 2026-02-20T21:17:59.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:17:59.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 143 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.7ba50050-0739-448b-a242-d41eab86b95a (peer_uuids:[]) 2026-02-20T21:17:59.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 144 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.e08f6bcb-8c6d-45f5-ad9d-1d0941546189 (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:17:59.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:17:59.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:17:59.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:17:59.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:17:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:17:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:17:59.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:17:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:17:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:17:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:17:59.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:17:59.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:17:59.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:56' 2026-02-20T21:17:59.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-20 21:17:56' 2026-02-20T21:17:59.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:17:59.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:59.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:59.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:17:59.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:17:56' 2026-02-20T21:17:59.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:17:59.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:17:59.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:17:59.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:17:56cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622275,"remote_snapshot_timestamp":1771622275,"replay_state":"idle"}2026-02-20 21:17:58' 2026-02-20T21:17:59.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:17:59.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:17:59.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:56' 2026-02-20T21:17:59.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:17:59.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:17:59.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:17:59.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:17:59.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:17:59.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:59.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:17:59.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:17:59.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:17:59.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:17:59.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:17:59.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:17:59.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:17:59.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:17:59.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:59.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:18:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:18:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:00.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:00.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:00.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:18:00.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:00.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:00.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:00.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:00.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:18:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:18:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:18:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:18:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622275,"remote_snapshot_timestamp":1771622275,"replay_state":"idle"} 2026-02-20T21:18:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:18:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:58 2026-02-20T21:18:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:18:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:18:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:18:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:18:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:17:56' 2026-02-20T21:18:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:18:00.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:18:00.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:18:00.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622275,"remote_snapshot_timestamp":1771622275,"replay_state":"idle"} 2026-02-20T21:18:00.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:18:00.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:18:00.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:18:00.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:18:00.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:18:00.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:18:00.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:18:00.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:18:00.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:58' 2026-02-20T21:18:00.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:17:58' 2026-02-20T21:18:00.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:00.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:00.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:00.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:18:00.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:17:58' 2026-02-20T21:18:00.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:00.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:00.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:18:00.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622275,"remote_snapshot_timestamp":1771622275,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:17:58cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:17:56' 2026-02-20T21:18:00.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:18:00.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:18:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:17:58' 2026-02-20T21:18:00.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:18:00.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:18:00.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:18:00.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:18:00.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:18:00.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:00.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:00.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:18:00.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test 2026-02-20T21:18:00.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:00.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:00.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:00.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-20T21:18:01.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:18:01.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:18:01.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:01.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:01.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:18:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:18:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:18:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:18:02.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:18:02.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:02.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:02.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:02.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:18:02.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:18:02.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:18:02.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:18:02.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:18:02.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:18:02.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:02.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:18:02.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:18:02.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:18:02.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:02.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:18:02.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:18:02.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:18:02.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:02.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:18:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:18:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:18:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:18:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:18:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:18:02.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:18:02.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:18:02.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:18:02.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:18:02.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:02.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:18:02.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:18:02.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:18:02.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:02.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:18:02.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:18:02.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:02.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:02.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:18:02.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:02.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:18:02.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:18:02.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:18:02.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:18:02.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:18:02.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:18:02.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:02.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:02.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:02.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:18:03.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:18:03.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:18:03.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:18:03.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:18:03.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:18:03.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:03.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:18:03.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:03.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:03.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:03.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:18:03.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:03.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:03.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:03.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:18:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:18:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:18:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:04.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:18:04.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:18:04.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:18:04.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:18:04.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:18:04.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:18:04.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:03 2026-02-20T21:18:04.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:18:04.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:18:04.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:18:04.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:18:04.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:01' 2026-02-20T21:18:04.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:18:04.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:18:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:18:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:18:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:18:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:18:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:18:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:18:04.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:18:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:18:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:18:04.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:18:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:03' 2026-02-20T21:18:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:18:03' 2026-02-20T21:18:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:18:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:18:03' 2026-02-20T21:18:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:18:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:18:03cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:18:01' 2026-02-20T21:18:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:18:04.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:18:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:03' 2026-02-20T21:18:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:18:04.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:18:04.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:18:04.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:18:04.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:18:04.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:04.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:04.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:18:04.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:04.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:04.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:04.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:18:04.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:04.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:04.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:04.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:18:05.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:18:05.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:05.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:18:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:05.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:18:05.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:18:05.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:18:05.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:18:05.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:18:05.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:18:05.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:01 2026-02-20T21:18:05.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:18:05.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:18:05.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:18:05.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:18:05.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:03' 2026-02-20T21:18:05.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:18:05.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:18:05.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:18:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:18:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:18:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:18:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:18:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:18:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:18:05.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:18:05.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:18:05.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:18:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:01' 2026-02-20T21:18:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:18:01' 2026-02-20T21:18:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:18:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:18:01' 2026-02-20T21:18:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:05.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:05.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:18:05.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:18:01cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:18:03' 2026-02-20T21:18:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:18:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:18:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:01' 2026-02-20T21:18:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:18:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:18:05.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:18:05.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:18:05.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:18:05.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test 2026-02-20T21:18:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:18:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-20T21:18:06.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:18:06.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-20T21:18:06.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:06.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:06.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:06.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-20T21:18:06.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:06.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:06.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:06.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:18:06.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:18:06.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:06.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:18:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-20T21:18:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:18:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:18:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:18:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:18:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:18:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:18:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:18:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:18:07.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:18:07.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:07.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:18:07.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:18:07.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:18:07.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:07.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:18:07.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:18:07.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:07.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:07.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:18:07.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:18:07.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:07.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:07.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:18:07.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:18:07.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:18:07.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:18:07.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:07.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:18:07.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:07.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:18:07.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:18:07.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:18:07.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:07.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:18:07.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:18:07.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:07.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:07.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:07.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:07.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:18:07.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:18:07.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:18:07.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:18:07.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:18:07.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:18:07.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:18:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:18:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:18:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:18:08.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:18:08.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:18:08.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:18:08.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:08.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:18:08.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:18:08.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:08.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:08.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:18:08.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:18:08.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:08.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:08.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:18:08.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:18:08.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:18:08.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:18:08.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:08.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:18:08.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:18:08.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:18:08.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:18:08.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:18:08.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:08.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:18:08.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:18:08.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:08.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:08.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:08.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:18:08.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:18:08.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:18:08.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:18:08.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:18:08.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:18:08.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:18:08.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:18:08.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:18:08.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:18:08.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:18:08.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:18:08.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:18:08.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:18:08.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:18:08.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:18:08.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:18:08.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:18:08.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:08.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:18:08.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:18:08.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:08.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:18:08.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:18:08.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:08.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:18:08.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:18:08.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster1 mirror test 2026-02-20T21:18:08.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:08.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:08.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:08.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/test 2026-02-20T21:18:11.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 147 2026-02-20T21:18:11.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror mirror test 2026-02-20T21:18:11.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster2 2026-02-20T21:18:11.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:11.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:18:11.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:18:11.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:18:11.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:18:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:18:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:18:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:18:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:18:11.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:18:11.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:18:11.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:18:11.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:11.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:11.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:11.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:18:11.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:18:11.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:18:11.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:18:11.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=147 2026-02-20T21:18:11.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:18:11.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:18:11.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:18:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:18:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:18:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:18:12.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:18:12.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:18:12.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=147 2026-02-20T21:18:12.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 147 = 147 2026-02-20T21:18:12.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:12.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-20T21:18:12.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:12.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:12.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:12.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:18:12.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:12.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:12.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:12.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:18:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-20T21:18:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:18:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:13.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:18:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:18:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:18:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:18:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:18:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:18:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:13 2026-02-20T21:18:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:18:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:18:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:18:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622285,"replay_state":"idle"} 2026-02-20T21:18:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:06 2026-02-20T21:18:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:18:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 147 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.177391d3-d3c7-494c-9440-277194259d86 (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:18:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:18:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:18:13.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:18:13.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:18:13.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:18:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:18:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:18:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:18:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:18:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:18:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:18:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:18:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:13' 2026-02-20T21:18:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-20 21:18:13' 2026-02-20T21:18:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:18:13.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:18:13' 2026-02-20T21:18:13.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:13.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:13.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:18:13.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary44044409mirror.0trial1992026-02-20 21:18:13cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622285,"replay_state":"idle"}2026-02-20 21:18:06' 2026-02-20T21:18:13.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:18:13.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:18:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:13' 2026-02-20T21:18:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:18:13.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:18:13.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:18:13.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:18:13.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:18:13.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:13.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:13.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-20T21:18:13.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:13.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:13.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:13.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:18:13.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:13.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:13.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:13.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:18:14.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-20T21:18:14.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:14.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:14.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:14.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:18:14.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:18:14.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:18:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:18:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:18:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622285,"replay_state":"idle"} 2026-02-20T21:18:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:18:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:06 2026-02-20T21:18:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:18:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:18:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:18:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:18:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:13' 2026-02-20T21:18:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:18:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:18:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:18:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622285,"replay_state":"idle"} 2026-02-20T21:18:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:18:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:18:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:18:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:18:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:18:14.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:18:14.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:18:14.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:18:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:06' 2026-02-20T21:18:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-20 21:18:06' 2026-02-20T21:18:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:18:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:18:06' 2026-02-20T21:18:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:18:14.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622285,"replay_state":"idle"}45294534mirror.1trial1992026-02-20 21:18:06cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:18:13' 2026-02-20T21:18:14.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:18:14.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:18:14.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:06' 2026-02-20T21:18:14.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:18:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:18:14.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:18:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:18:14.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:18:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster1 mirror test 2026-02-20T21:18:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:14.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-20T21:18:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:18:14.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-20T21:18:14.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:14.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:14.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:14.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-20T21:18:14.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:14.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:14.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:14.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:18:14.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:18:14.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:14.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:18:15.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:18:15.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:15.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:15.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:15.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:18:15.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:18:15.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:18:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:18:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:18:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:18:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:18:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:18:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:18:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:18:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:18:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:18:15.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:15.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:18:15.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:18:15.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:15.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:15.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:18:15.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:18:15.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:15.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:15.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:18:15.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:18:15.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:18:15.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:18:15.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:15.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:18:15.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:15.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:18:15.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:18:15.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:18:15.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:15.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:18:15.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:18:15.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:15.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:15.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:15.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:15.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:18:15.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:18:15.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:18:15.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:18:15.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:18:15.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:18:15.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:18:15.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:18:15.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:18:15.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:18:16.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:18:16.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:18:16.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:18:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:18:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:18:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:18:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:18:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:18:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:18:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:18:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:18:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:18:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:18:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:18:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:18:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:18:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:18:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:18:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:18:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:18:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:18:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:18:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:18:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:18:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:18:16.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:18:16.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:18:16.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:18:16.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:18:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:18:16.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:18:16.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:18:16.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:18:16.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:18:16.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:18:16.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = started 2026-02-20T21:18:16.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:16.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:18:18.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-20T21:18:18.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:18.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:18.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:18.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:18:18.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:18:18.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:18:18.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-20T21:18:18.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:18:18.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:18:18.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:18.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:18:18.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:18:18.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:18:18.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:18.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:18:18.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:18:18.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:18:18.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:18.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-20T21:18:18.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:18:18.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:18.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:18.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:18:18.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:18:18.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:18.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:18.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:18:18.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:18:18.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:18:18.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:18:18.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:18.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:18:18.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:18.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:18:18.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:18:18.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:18:18.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:18.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:18:18.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:18:18.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:18.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:18.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:18.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:18.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:18:18.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:18:18.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:18:18.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:18:18.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:18:18.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:18:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:18:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:18:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:18:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:18:18.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:18:18.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:18:18.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:18:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test 2026-02-20T21:18:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:18:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:18:18.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:18:18.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:18.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:18.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:18:18.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:18:18.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:18:18.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:18:18.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:18.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:18:18.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:18:18.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:18:18.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:18:18.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:18:18.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:18.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:18:18.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:18:18.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:18.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:18.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:18.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:18:18.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:18:18.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:18:18.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:18:18.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:18:18.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:18:18.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:18:18.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:18:18.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:18:18.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:18:18.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:18:18.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:18:18.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:18:18.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:18:18.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:18:18.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:18:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-20T21:18:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:18.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:18:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:18:19.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-20T21:18:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:18:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:18:19.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:18:19.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:18:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:18:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:18:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:18:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:18 2026-02-20T21:18:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:18:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:18:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:18:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:18:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:16' 2026-02-20T21:18:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:18:19.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:18:19.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:18:19.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:18:19.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:18:19.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:18:19.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:18:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:18:19.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:18:19.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:18:19.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:18:19.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:18:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:18' 2026-02-20T21:18:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-20 21:18:18' 2026-02-20T21:18:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:19.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:18:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:18:18' 2026-02-20T21:18:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:18:19.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary44044409mirror.0trial1992026-02-20 21:18:18cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:18:16' 2026-02-20T21:18:19.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:18:19.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:18:19.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:18' 2026-02-20T21:18:19.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:18:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:18:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:18:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:18:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:18:19.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:19.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:19.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-20T21:18:19.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:19.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:19.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:19.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:18:19.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:19.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:19.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:19.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:18:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-20T21:18:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:18:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:18:20.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:18:20.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:18:20.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:18:20.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:18:20.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:18:20.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:16 2026-02-20T21:18:20.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:18:20.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:18:20.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:18:20.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:18:20.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:18' 2026-02-20T21:18:20.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:18:20.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:18:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:18:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:18:20.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:18:20.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:18:20.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:18:20.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:18:20.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:18:20.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:18:20.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:18:20.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:18:20.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:16' 2026-02-20T21:18:20.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-20 21:18:16' 2026-02-20T21:18:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:18:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:18:16' 2026-02-20T21:18:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:18:20.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+unknownremote image is not primary45294534mirror.1trial1992026-02-20 21:18:16cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:18:18' 2026-02-20T21:18:20.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:18:20.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:18:20.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:16' 2026-02-20T21:18:20.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:18:20.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:18:20.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:18:20.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:18:20.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:18:20.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster2 mirror test 2026-02-20T21:18:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:18:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-20T21:18:21.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:18:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:18:21.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:21.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:21.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:21.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:18:21.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:21.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:21.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:21.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:18:21.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:18:21.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:21.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:18:22.937 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:18:22.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:22.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:22.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:22.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:18:22.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:18:22.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:18:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:18:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:18:22.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:18:22.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:22.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:18:22.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:18:22.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:18:22.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:22.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:18:22.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:18:22.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:18:22.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:22.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:18:22.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:18:22.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:22.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:22.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:18:22.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:18:22.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:22.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:22.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:18:22.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:18:22.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:18:22.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:18:22.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:18:22.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:18:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:18:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:18:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:18:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:18:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:18:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:18:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:18:22.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:18:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:18:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:18:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:18:22.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:18:22.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:22.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:22.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:22.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:18:22.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:18:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:18:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:18:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:18:23.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:18:23.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:23.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:18:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:18:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:18:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:18:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:18:25.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:18:25.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:18:25.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:18:25.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:25.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:18:25.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:18:25.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:18:25.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:25.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:18:25.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:18:25.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:18:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:18:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:18:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:18:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:18:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:18:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:18:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:18:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:18:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:18:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:18:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:18:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:18:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:18:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:18:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:18:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:18:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:25.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:18:25.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:18:25.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:18:25.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:18:25.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:18:25.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:18:25.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:25.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:25.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:25.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:18:25.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:18:25.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:18:25.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:18:25.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:18:25.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:18:25.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:18:25.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:18:25.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:25.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:18:25.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:18:25.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:25.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:18:25.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:18:25.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:25.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:18:25.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:18:25.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:18:25.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:25.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:25.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:25.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:18:28.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 150 2026-02-20T21:18:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:18:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:18:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:18:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:18:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:18:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:18:28.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:18:28.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:18:28.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:18:28.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:18:28.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:18:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:18:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:18:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:28.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:28.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:28.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.status 2026-02-20T21:18:28.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:18:28.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:18:28.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:18:28.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=150 2026-02-20T21:18:28.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:18:28.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:18:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:18:28.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:18:28.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:28.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:28.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:28.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.status 2026-02-20T21:18:28.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:18:28.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:18:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:18:28.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=150 2026-02-20T21:18:28.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 150 = 150 2026-02-20T21:18:28.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:28.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-20T21:18:28.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:28.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:28.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:28.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:18:28.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:28.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:28.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:28.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:18:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-20T21:18:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:18:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:29.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:29.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-20T21:18:29.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:18:29.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:18:29.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:18:29.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:18:29.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:18:29.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:26 2026-02-20T21:18:29.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:18:29.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:18:29.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:18:29.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"} 2026-02-20T21:18:29.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:28 2026-02-20T21:18:29.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:18:29.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 150 .mirror.primary.88d69193-4691-4616-9634-b744c1077c1e.00f46666-e100-4b39-b5b7-4b4d8f051ee5 (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:18:29.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:18:29.440 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:18:29.440 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:18:29.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:18:29.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:18:29.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:18:29.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:18:29.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:18:29.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:18:29.443 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:18:29.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:18:29.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:18:29.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:26' 2026-02-20T21:18:29.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-20 21:18:26' 2026-02-20T21:18:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:18:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:18:26' 2026-02-20T21:18:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:18:29.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:18:26cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}2026-02-20 21:18:28' 2026-02-20T21:18:29.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:18:29.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:18:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:26' 2026-02-20T21:18:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:18:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:18:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:18:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:18:29.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:18:29.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:29.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:29.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:18:29.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:29.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:29.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:29.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:18:29.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:29.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:29.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:29.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:18:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:18:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:18:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:18:30.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:18:30.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 88d69193-4691-4616-9634-b744c1077c1e 2026-02-20T21:18:30.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:18:30.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"} 2026-02-20T21:18:30.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:18:30.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:28 2026-02-20T21:18:30.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:18:30.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:18:30.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:18:30.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:18:30.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:26' 2026-02-20T21:18:30.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:18:30.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:18:30.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:18:30.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"} 2026-02-20T21:18:30.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:18:30.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:18:30.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:18:30.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:18:30.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:18:30.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:18:30.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:18:30.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:18:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:28' 2026-02-20T21:18:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:18:28' 2026-02-20T21:18:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:18:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:18:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:18:28' 2026-02-20T21:18:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:18:30.554 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:18:28cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:18:26' 2026-02-20T21:18:30.554 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:18:30.554 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:18:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:28' 2026-02-20T21:18:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:18:30.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:18:30.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:18:30.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:18:30.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:18:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:18:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ count_mirror_snaps cluster1 mirror test 2026-02-20T21:18:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:18:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:18:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:18:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 snap ls mirror/test --all 2026-02-20T21:18:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -c -F ' mirror (' 2026-02-20T21:18:30.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 3 -le 3 2026-02-20T21:18:30.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ count_mirror_snaps cluster2 mirror test 2026-02-20T21:18:30.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:18:30.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:18:30.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:18:30.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 snap ls mirror/test --all 2026-02-20T21:18:30.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -c -F ' mirror (' 2026-02-20T21:18:30.819 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 1 -le 3 2026-02-20T21:18:30.819 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: force promote' 2026-02-20T21:18:30.819 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:18:30.819 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:18:30.821 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:18:30 TEST: force promote 2026-02-20T21:18:30.821 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:18:30 TEST: force promote 2026-02-20T21:18:30.821 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ force_promote_image=test_force_promote 2026-02-20T21:18:30.821 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image_and_enable_mirror cluster2 mirror test_force_promote snapshot 2026-02-20T21:18:30.821 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:30.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:18:30.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:30.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:18:30.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:30.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:18:30.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:18:30.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:18:30.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:18:30.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image cluster2 mirror test_force_promote 2026-02-20T21:18:30.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:30.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:18:30.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:30.823 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:18:30.823 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:30.823 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:18:30.823 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=128 2026-02-20T21:18:30.823 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:18:30.823 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,object-map,fast-diff mirror/test_force_promote 2026-02-20T21:18:30.849 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:18:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:18:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror test_force_promote snapshot 2026-02-20T21:18:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:18:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_force_promote snapshot 2026-02-20T21:18:31.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:18:31.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/test_force_promote 2026-02-20T21:18:31.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test_force_promote': 2026-02-20T21:18:31.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:18:31.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:18:31.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:18:31.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 1559d5adaded 2026-02-20T21:18:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.1559d5adaded 2026-02-20T21:18:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:18:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:18:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:18:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:18:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:18:30 2026 2026-02-20T21:18:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:18:30 2026 2026-02-20T21:18:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:18:30 2026 2026-02-20T21:18:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:18:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:18:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 081cd46a-ad2d-4d9c-bf4b-db62b403c3f6 2026-02-20T21:18:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:18:31.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test_force_promote 100 2026-02-20T21:18:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:18:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:18:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:18:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:18:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:18:31.567 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:18:31.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:18:31.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 4761.98 bytes/sec: 19 MiB/s 2026-02-20T21:18:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_force_promote 2026-02-20T21:18:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test_force_promote stopped 2026-02-20T21:18:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:18:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:18:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:18:32.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test_force_promote stopped 2026-02-20T21:18:32.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:32.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:32.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:32.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:18:32.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:18:32.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:18:32.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_force_promote 2026-02-20T21:18:32.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:18:32.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:32.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:18:32.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:18:32.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:18:32.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:18:32.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:32.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:18:32.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:18:32.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:18:32.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:32.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote 2026-02-20T21:18:32.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:18:32.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:32.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:32.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:18:32.601 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:18:32.601 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:32.601 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:32.601 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:18:32.601 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:18:32.601 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:18:32.601 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:18:32.601 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:32.601 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:18:32.601 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:32.601 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:18:32.602 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:18:32.602 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:18:32.602 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:32.602 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:18:32.602 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:18:32.602 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:32.602 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:32.602 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:32.602 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:32.602 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:18:32.602 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:18:32.603 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:18:32.603 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:18:32.603 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:18:32.603 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:18:32.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:18:32.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:18:32.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:18:32.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-02-20T21:18:32.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-20T21:18:32.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:18:32.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:18:32.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:32.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_force_promote 2026-02-20T21:18:32.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:18:32.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:32.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:32.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:18:32.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:18:32.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:18:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:18:32.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:18:32.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:18:32.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:18:32.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_force_promote 2026-02-20T21:18:32.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-20T21:18:32.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:18:32.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:18:32.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:32.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_force_promote 2026-02-20T21:18:32.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:2 2026-02-20T21:18:32.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:32.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:2 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:2 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:2 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:18:32.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-20T21:18:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:18:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:18:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:18:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_force_promote 2026-02-20T21:18:32.758 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-20T21:18:32.758 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:18:32.758 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:18:32.759 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:32.759 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_force_promote 2026-02-20T21:18:32.759 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:3 2026-02-20T21:18:32.759 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:32.759 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:32.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-20T21:18:32.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:3 2026-02-20T21:18:32.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:32.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:32.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:18:32.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:18:32.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-20T21:18:32.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:18:32.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:32.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:18:32.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:18:32.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-02-20T21:18:32.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:3 2026-02-20T21:18:32.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:18:32.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:32.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-20T21:18:32.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:3 2026-02-20T21:18:32.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:32.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:32.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:32.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:18:32.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-20T21:18:32.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:18:32.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:18:32.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:18:32.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:18:32.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-20T21:18:32.771 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:18:32.772 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:18:32.772 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:18:32.772 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_force_promote 2026-02-20T21:18:32.812 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-20T21:18:32.812 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:18:32.812 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:18:32.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:32.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 1 2026-02-20T21:18:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:18:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:18:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote 2026-02-20T21:18:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:18:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:18:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:18:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:18:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:18:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:18:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:18:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:18:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:18:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:18:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:18:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:18:33.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:18:33.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:33.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:33.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:33.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:33.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:18:33.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:18:33.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:18:33.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:18:33.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:18:33.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:18:33.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:18:33.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:18:33.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:18:33.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-02-20T21:18:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-20T21:18:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:18:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:18:33.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:33.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_force_promote 2026-02-20T21:18:33.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:18:33.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:33.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:33.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:18:33.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:18:33.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:33.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:33.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:18:33.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:18:33.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:18:33.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:18:33.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:33.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:18:33.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:18:33.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:18:33.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:18:33.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:18:33.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:18:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:18:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:18:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:18:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:18:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:18:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:18:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:18:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:18:33.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:18:33.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:18:33.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:18:33.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_force_promote 2026-02-20T21:18:33.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-20T21:18:33.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:18:33.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:18:33.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:33.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_force_promote 2026-02-20T21:18:33.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:2 2026-02-20T21:18:33.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:33.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:33.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-20T21:18:33.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:2 2026-02-20T21:18:33.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:33.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:33.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:18:33.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:18:33.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-20T21:18:33.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:18:33.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:33.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:18:33.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:18:33.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-20T21:18:33.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:2 2026-02-20T21:18:33.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:18:33.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:33.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-20T21:18:33.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:2 2026-02-20T21:18:33.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:33.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:33.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:33.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:18:33.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-20T21:18:33.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:18:33.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:18:33.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:18:33.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:18:33.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-20T21:18:33.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:18:33.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:18:33.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:18:33.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_force_promote 2026-02-20T21:18:33.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:18:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:18:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:18:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:18:33.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:18:33.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:33.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test_force_promote 2026-02-20T21:18:33.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:33.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:33.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:33.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test_force_promote started 2026-02-20T21:18:33.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:33.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:33.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:33.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:18:33.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:18:33.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:33.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:18:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test_force_promote started 2026-02-20T21:18:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:18:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:18:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:18:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_force_promote 2026-02-20T21:18:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:18:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:18:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:18:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:18:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:18:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:18:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:18:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:18:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote 2026-02-20T21:18:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:18:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:18:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:18:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:18:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:18:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:18:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:18:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:18:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:18:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:18:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:18:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:18:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:18:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:18:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:18:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:18:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:18:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:18:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:18:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:18:34.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:18:35.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:35.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:35.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:35.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-02-20T21:18:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-20T21:18:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:18:35.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:18:35.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:35.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote 2026-02-20T21:18:35.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:18:35.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:35.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:35.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:18:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:18:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:18:35.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:18:35.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:18:35.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote 2026-02-20T21:18:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:18:35.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:18:35.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:18:35.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:18:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:18:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:18:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:18:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test_force_promote 2026-02-20T21:18:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:18:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:18:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:18:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:18:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:18:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test_force_promote 2026-02-20T21:18:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_force_promote 2026-02-20T21:18:35.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 152 2026-02-20T21:18:35.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test_force_promote 2026-02-20T21:18:35.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:18:35.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:35.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:18:35.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:18:35.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:35.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test_force_promote.status 2026-02-20T21:18:35.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test_force_promote.status 2026-02-20T21:18:35.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:18:35.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_force_promote.status 2026-02-20T21:18:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test_force_promote.status 2026-02-20T21:18:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test_force_promote.status 2026-02-20T21:18:35.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:18:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_force_promote.status 2026-02-20T21:18:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_force_promote /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_force_promote.status 2026-02-20T21:18:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:35.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:35.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:35.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_force_promote.status 2026-02-20T21:18:35.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_force_promote --format xml 2026-02-20T21:18:35.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:18:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:18:35.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=152 2026-02-20T21:18:35.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:18:35.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:18:35.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:18:35.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_force_promote /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_force_promote.status 2026-02-20T21:18:35.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:35.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:35.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:35.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_force_promote.status 2026-02-20T21:18:35.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_force_promote --format xml 2026-02-20T21:18:35.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:18:35.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:18:35.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=151 2026-02-20T21:18:35.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 152 = 151 2026-02-20T21:18:35.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:18:35.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.4 2026-02-20T21:18:35.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_force_promote /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_force_promote.status 2026-02-20T21:18:35.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:35.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:35.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:35.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_force_promote.status 2026-02-20T21:18:35.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_force_promote --format xml 2026-02-20T21:18:35.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:18:35.908 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:18:35.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=151 2026-02-20T21:18:35.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 152 = 151 2026-02-20T21:18:35.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:18:35.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.8 2026-02-20T21:18:36.710 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_force_promote /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_force_promote.status 2026-02-20T21:18:36.710 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:36.710 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:36.710 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:36.710 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_force_promote.status 2026-02-20T21:18:36.710 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_force_promote --format xml 2026-02-20T21:18:36.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:18:36.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:18:36.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=152 2026-02-20T21:18:36.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 152 = 152 2026-02-20T21:18:36.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:36.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_force_promote 2026-02-20T21:18:36.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:36.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:36.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:36.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:18:36.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_force_promote up+replaying 2026-02-20T21:18:36.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:36.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:36.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:36.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:18:36.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:36.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:36.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:36.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:18:37.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_force_promote up+replaying '' '' 2026-02-20T21:18:37.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:37.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:37.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:37.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:18:37.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:37.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:37.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:37.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:37.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_force_promote 2026-02-20T21:18:37.780 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_force_promote: 2026-02-20T21:18:37.780 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 081cd46a-ad2d-4d9c-bf4b-db62b403c3f6 2026-02-20T21:18:37.780 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:18:37.780 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622311,"replay_state":"idle"} 2026-02-20T21:18:37.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:18:37.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:33 2026-02-20T21:18:37.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:18:37.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:18:37.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:18:37.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:18:37.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:36' 2026-02-20T21:18:37.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:18:37.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:18:37.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:18:37.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622311,"replay_state":"idle"} 2026-02-20T21:18:37.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:18:37.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:18:37.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:18:37.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:18:37.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:18:37.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:18:37.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:18:37.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:18:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:33' 2026-02-20T21:18:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_force_promote up+replaying '2026-02-20 21:18:33' 2026-02-20T21:18:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:18:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:18:33' 2026-02-20T21:18:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:18:37.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:18:28cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:18:36test_force_promote081cd46a-ad2d-4d9c-bf4b-db62b403c3f6up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622311,"replay_state":"idle"}44114415mirror.1trial1992026-02-20 21:18:33cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:18:36' 2026-02-20T21:18:37.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:18:37.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/last_update' 2026-02-20T21:18:37.821 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:33' 2026-02-20T21:18:37.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/state' 2026-02-20T21:18:37.823 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:18:37.823 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:18:37.824 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:18:37.824 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:18:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_force_promote up+stopped 2026-02-20T21:18:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:18:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:18:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test_force_promote up+stopped '' '' 2026-02-20T21:18:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:18:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_force_promote 2026-02-20T21:18:38.858 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_force_promote: 2026-02-20T21:18:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 081cd46a-ad2d-4d9c-bf4b-db62b403c3f6 2026-02-20T21:18:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:18:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:18:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.2 on trial199 2026-02-20T21:18:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:36 2026-02-20T21:18:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:18:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:18:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:18:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622311,"replay_state":"idle"} 2026-02-20T21:18:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:33 2026-02-20T21:18:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:18:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 151 .mirror.primary.081cd46a-ad2d-4d9c-bf4b-db62b403c3f6.3ba79432-6862-43bc-a7fb-59021bd10d9f (peer_uuids:[]) 2026-02-20T21:18:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 152 .mirror.primary.081cd46a-ad2d-4d9c-bf4b-db62b403c3f6.cec8e5cd-39de-4302-b731-4231faf807e4 (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:18:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:18:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:18:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:18:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:18:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:18:38.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:18:38.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:18:38.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:18:38.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:18:38.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.2 on trial199 2026-02-20T21:18:38.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:18:38.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:18:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:36' 2026-02-20T21:18:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_force_promote up+stopped '2026-02-20 21:18:36' 2026-02-20T21:18:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:18:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:18:36' 2026-02-20T21:18:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:18:38.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK245294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:18:36cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}2026-02-20 21:18:28test_force_promote081cd46a-ad2d-4d9c-bf4b-db62b403c3f6up+stoppedlocal image is primary45364539mirror.2trial1992026-02-20 21:18:36cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622311,"replay_state":"idle"}2026-02-20 21:18:33' 2026-02-20T21:18:38.896 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:18:38.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/last_update' 2026-02-20T21:18:38.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:36' 2026-02-20T21:18:38.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/state' 2026-02-20T21:18:38.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:18:38.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:18:38.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:18:38.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:18:38.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:38.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:38.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test_force_promote --force 2026-02-20T21:18:38.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:38.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:38.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:38.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force=--force 2026-02-20T21:18:38.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_force_promote --force 2026-02-20T21:18:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:18:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test_force_promote 2026-02-20T21:18:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test_force_promote stopped 2026-02-20T21:18:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:18:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:18:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:46.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:18:47.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test_force_promote stopped 2026-02-20T21:18:47.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:47.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:47.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:47.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:18:47.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:18:47.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:18:47.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_force_promote 2026-02-20T21:18:47.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:18:47.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:18:47.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:47.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:18:47.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:18:47.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:18:47.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:47.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:18:47.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:18:47.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:18:47.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:47.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote 2026-02-20T21:18:47.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:18:47.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:47.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:47.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:18:47.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:18:47.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:47.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:47.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:18:47.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:18:47.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:18:47.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:18:47.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:18:47.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:18:47.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:47.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:18:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:18:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:18:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:18:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:18:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:18:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:18:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:18:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:18:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:18:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:18:48.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:18:48.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:48.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:48.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:48.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-02-20T21:18:48.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-20T21:18:48.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:18:48.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:18:48.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:48.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote 2026-02-20T21:18:48.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:18:48.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:48.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:18:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:18:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:18:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:18:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:18:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:18:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:18:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:18:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:18:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:18:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:18:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:18:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:18:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:18:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:18:48.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:18:48.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:18:48.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:18:48.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:18:48.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:18:48.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:18:48.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:18:48.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:18:48.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:18:48.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:18:48.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote 2026-02-20T21:18:48.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:18:48.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:18:48.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:18:48.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:18:48.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:18:48.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:48.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_force_promote 2026-02-20T21:18:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test_force_promote stopped 2026-02-20T21:18:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:18:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:18:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:18:49.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test_force_promote stopped 2026-02-20T21:18:49.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:49.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:49.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:49.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:18:49.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:18:49.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:18:49.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_force_promote 2026-02-20T21:18:49.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:18:49.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:49.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:18:49.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:18:49.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:18:49.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:49.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:18:49.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:18:49.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:18:49.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:18:49.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:49.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote 2026-02-20T21:18:49.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:18:49.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:49.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:49.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:18:49.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:18:49.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:49.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:49.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:18:49.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:18:49.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:18:49.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:18:49.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:49.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:18:49.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:49.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:18:49.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:18:49.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:18:49.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:49.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:18:49.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:18:49.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:49.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:49.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:49.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:49.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:18:49.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:18:49.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:18:49.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:18:49.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:18:49.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:18:49.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:18:49.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:18:49.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:18:49.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-02-20T21:18:49.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-20T21:18:49.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:18:49.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:18:49.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:49.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_force_promote 2026-02-20T21:18:49.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:18:49.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:49.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:49.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:18:49.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:18:49.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:49.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:49.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:18:49.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:18:49.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:18:49.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:18:49.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:49.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:18:49.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:18:49.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:18:49.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:18:49.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:18:49.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:49.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:18:49.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:18:49.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:49.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:49.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:49.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:18:49.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:18:49.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:18:49.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:18:49.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:18:49.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:18:49.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:18:49.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:18:49.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:18:49.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:18:49.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_force_promote 2026-02-20T21:18:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-20T21:18:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:18:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:18:49.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:49.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_force_promote 2026-02-20T21:18:49.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:2 2026-02-20T21:18:49.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:49.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:49.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-20T21:18:49.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:2 2026-02-20T21:18:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:18:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:18:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-20T21:18:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:18:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:18:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:18:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-20T21:18:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:2 2026-02-20T21:18:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:18:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:49.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-20T21:18:49.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:2 2026-02-20T21:18:49.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:49.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:49.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:18:49.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:18:49.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-20T21:18:49.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:18:49.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:18:49.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:18:49.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:18:49.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-20T21:18:49.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:18:49.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:18:49.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:18:49.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_force_promote 2026-02-20T21:18:49.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:18:49.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:18:49.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:18:49.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:18:49.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:18:49.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:49.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_force_promote up+stopped 2026-02-20T21:18:49.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:49.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:49.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:49.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:18:49.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:49.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:49.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:49.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:18:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_force_promote up+stopped '' '' 2026-02-20T21:18:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:18:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:50.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_force_promote 2026-02-20T21:18:50.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_force_promote: 2026-02-20T21:18:50.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 081cd46a-ad2d-4d9c-bf4b-db62b403c3f6 2026-02-20T21:18:50.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:18:50.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:18:50.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:18:50.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:48 2026-02-20T21:18:50.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:18:50.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:18:50.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:18:50.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:18:50.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:46 2026-02-20T21:18:50.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:18:50.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 154 .mirror.primary.081cd46a-ad2d-4d9c-bf4b-db62b403c3f6.7fad085a-a796-4b48-b66b-159928124e47 (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:18:50.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:18:50.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:18:50.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:18:50.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:18:50.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:18:50.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:18:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:18:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:18:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:18:50.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:18:50.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:18:50.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:18:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:48' 2026-02-20T21:18:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_force_promote up+stopped '2026-02-20 21:18:48' 2026-02-20T21:18:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:50.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:50.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:18:50.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:18:48' 2026-02-20T21:18:50.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:50.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:50.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:18:50.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK1144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:18:28cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:18:46test_force_promote081cd46a-ad2d-4d9c-bf4b-db62b403c3f6up+stoppedlocal image is primary44114415mirror.1trial1992026-02-20 21:18:48cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:18:46' 2026-02-20T21:18:50.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:18:50.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/last_update' 2026-02-20T21:18:50.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:48' 2026-02-20T21:18:50.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/state' 2026-02-20T21:18:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:18:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:18:50.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:18:50.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:18:50.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:50.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:50.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_force_promote up+stopped 2026-02-20T21:18:50.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:50.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:50.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:50.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:18:50.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:50.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:50.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:50.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:18:51.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test_force_promote up+stopped '' '' 2026-02-20T21:18:51.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:18:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:18:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:18:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:51.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_force_promote 2026-02-20T21:18:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_force_promote: 2026-02-20T21:18:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 081cd46a-ad2d-4d9c-bf4b-db62b403c3f6 2026-02-20T21:18:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:18:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:18:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.2 on trial199 2026-02-20T21:18:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:46 2026-02-20T21:18:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:18:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:18:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:18:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:18:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:48 2026-02-20T21:18:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:18:51.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 151 .mirror.primary.081cd46a-ad2d-4d9c-bf4b-db62b403c3f6.3ba79432-6862-43bc-a7fb-59021bd10d9f (peer_uuids:[]) 2026-02-20T21:18:51.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 152 .mirror.primary.081cd46a-ad2d-4d9c-bf4b-db62b403c3f6.cec8e5cd-39de-4302-b731-4231faf807e4 (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:18:51.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:18:51.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:18:51.408 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:18:51.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:18:51.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:18:51.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:18:51.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:18:51.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:18:51.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:18:51.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.2 on trial199 2026-02-20T21:18:51.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:18:51.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:18:51.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:46' 2026-02-20T21:18:51.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_force_promote up+stopped '2026-02-20 21:18:46' 2026-02-20T21:18:51.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:51.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:51.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:51.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:18:51.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:18:46' 2026-02-20T21:18:51.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:18:51.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:18:51.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:18:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK1145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:18:46cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}2026-02-20 21:18:28test_force_promote081cd46a-ad2d-4d9c-bf4b-db62b403c3f6up+stoppedlocal image is primary45364539mirror.2trial1992026-02-20 21:18:46cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:18:48' 2026-02-20T21:18:51.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:18:51.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/last_update' 2026-02-20T21:18:51.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:46' 2026-02-20T21:18:51.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/state' 2026-02-20T21:18:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:18:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:18:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:18:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:18:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster1 mirror test_force_promote 100 2026-02-20T21:18:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:18:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:18:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:18:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:18:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 bench mirror/test_force_promote --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:18:51.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:18:51.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:18:51.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5000.08 bytes/sec: 20 MiB/s 2026-02-20T21:18:51.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test_force_promote 100 2026-02-20T21:18:51.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:51.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:51.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:51.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:18:51.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:18:51.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:18:51.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:18:51.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:18:51.541 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:18:51.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:18:51.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5555.64 bytes/sec: 22 MiB/s 2026-02-20T21:18:51.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster1 mirror test_force_promote 2026-02-20T21:18:51.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:51.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:51.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:51.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:18:51.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:18:51.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster1 mirror test_force_promote 2026-02-20T21:18:51.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:51.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:51.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:51.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_force_promote 2026-02-20T21:18:51.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 rm mirror/test_force_promote 2026-02-20T21:18:52.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:18:52.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:52.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster2 mirror test_force_promote 2026-02-20T21:18:52.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:52.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:52.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:18:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:18:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster2 mirror test_force_promote 2026-02-20T21:18:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote 2026-02-20T21:18:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_force_promote 2026-02-20T21:18:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rm mirror/test_force_promote 2026-02-20T21:18:53.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:18:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: cloned images' 2026-02-20T21:18:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:18:53.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:18:53.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:18:53 TEST: cloned images 2026-02-20T21:18:53.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:18:53 TEST: cloned images 2026-02-20T21:18:53.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog ' - default' 2026-02-20T21:18:53.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:18:53.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:18:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:18:53 - default 2026-02-20T21:18:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:18:53 - default 2026-02-20T21:18:53.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ parent_image=test_parent 2026-02-20T21:18:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ parent_snap=snap 2026-02-20T21:18:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image_and_enable_mirror cluster2 mirror_parent test_parent snapshot 2026-02-20T21:18:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:18:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:18:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:18:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:18:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:18:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:18:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:18:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:18:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image cluster2 mirror_parent test_parent 2026-02-20T21:18:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:18:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:18:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:18:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:18:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:18:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=128 2026-02-20T21:18:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:18:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,object-map,fast-diff mirror_parent/test_parent 2026-02-20T21:18:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror_parent 2026-02-20T21:18:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:18:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:18:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror_parent test_parent snapshot 2026-02-20T21:18:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:18:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:18:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:18:54.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror_parent/test_parent snapshot 2026-02-20T21:18:54.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:18:54.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror_parent/test_parent 2026-02-20T21:18:54.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test_parent': 2026-02-20T21:18:54.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:18:54.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:18:54.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:18:54.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 1580920ccef7 2026-02-20T21:18:54.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.1580920ccef7 2026-02-20T21:18:54.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:18:54.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:18:54.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:18:54.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:18:54.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:18:53 2026 2026-02-20T21:18:54.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:18:53 2026 2026-02-20T21:18:54.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:18:53 2026 2026-02-20T21:18:54.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:18:54.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:18:54.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 0dc6cc8f-9aed-4f34-9291-5356f16c50eb 2026-02-20T21:18:54.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:18:54.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror_parent test_parent 100 2026-02-20T21:18:54.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:54.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:18:54.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:18:54.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:18:54.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:18:54.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:18:54.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:18:54.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror_parent/test_parent --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:18:55.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:18:55.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:18:55.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5000.08 bytes/sec: 20 MiB/s 2026-02-20T21:18:55.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_snapshot cluster2 mirror_parent test_parent snap 2026-02-20T21:18:55.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:55.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:18:55.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:18:55.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap 2026-02-20T21:18:55.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap create mirror_parent/test_parent@snap 2026-02-20T21:18:55.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-20T21:18:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ protect_snapshot cluster2 mirror_parent test_parent snap 2026-02-20T21:18:55.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:55.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:18:55.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:18:55.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap 2026-02-20T21:18:55.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap protect mirror_parent/test_parent@snap 2026-02-20T21:18:55.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ clone_image=test_clone 2026-02-20T21:18:55.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ clone_image cluster2 mirror_parent test_parent snap mirror test_clone 2026-02-20T21:18:55.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:55.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_pool=mirror_parent 2026-02-20T21:18:55.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_image=test_parent 2026-02-20T21:18:55.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_snap=snap 2026-02-20T21:18:55.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local clone_pool=mirror 2026-02-20T21:18:55.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local clone_image=test_clone 2026-02-20T21:18:55.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 6 2026-02-20T21:18:55.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 clone mirror_parent/test_parent@snap mirror/test_clone --image-feature layering,exclusive-lock,object-map,fast-diff 2026-02-20T21:18:56.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test_clone 100 2026-02-20T21:18:56.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:56.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:56.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone 2026-02-20T21:18:56.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:18:56.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:18:56.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:18:56.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:18:56.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test_clone --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:18:56.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:18:56.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:18:56.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 6666.77 bytes/sec: 26 MiB/s 2026-02-20T21:18:56.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:18:56.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror test_clone snapshot 2026-02-20T21:18:56.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:56.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:18:56.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone 2026-02-20T21:18:56.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:18:56.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_clone snapshot 2026-02-20T21:18:56.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:18:56.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/test_clone 2026-02-20T21:18:56.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test_clone': 2026-02-20T21:18:56.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:18:56.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:18:56.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:18:56.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 159273de391a 2026-02-20T21:18:56.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.159273de391a 2026-02-20T21:18:56.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:18:56.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:18:56.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:18:56.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:18:56.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:18:56 2026 2026-02-20T21:18:56.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:18:56 2026 2026-02-20T21:18:56.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:18:56 2026 2026-02-20T21:18:56.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: parent: mirror_parent/test_parent@snap 2026-02-20T21:18:56.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: overlap: 128 MiB 2026-02-20T21:18:56.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:18:56.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:18:56.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 3027f03e-b1b6-46ff-8902-5ee7692d9b2e 2026-02-20T21:18:56.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:18:56.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror_parent test_parent 2026-02-20T21:18:56.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:56.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:18:56.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:18:56.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror_parent test_parent started 2026-02-20T21:18:56.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:56.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:18:56.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:18:56.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:18:56.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:18:56.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:56.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:18:57.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror_parent test_parent started 2026-02-20T21:18:57.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:57.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:18:57.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:18:57.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:18:57.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:18:57.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:18:57.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror_parent/test_parent 2026-02-20T21:18:57.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:18:57.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:57.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:18:57.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:18:57.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:18:57.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:18:57.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:57.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:18:57.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:18:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:18:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror_parent/test_parent 2026-02-20T21:18:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:18:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:18:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:18:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:18:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:18:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:18:57.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:18:57.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:18:57.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:18:57.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:18:57.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:18:57.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:18:57.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:18:57.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:58.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:18:58.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:18:58.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:18:58.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:18:58.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:18:58.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:18:58.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:18:58.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:18:58.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:18:58.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:18:58.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:18:58.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:18:58.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:18:58.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:18:58.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:18:58.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:18:58.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:58.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:58.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:58.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror_parent/test_parent 2026-02-20T21:18:58.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:18:58.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:18:58.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:18:58.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:18:58.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:18:58.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:18:58.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:18:58.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:18:58.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror_parent mirror_parent test_parent 2026-02-20T21:18:58.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:18:58.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:58.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror_parent 2026-02-20T21:18:58.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror_parent 2026-02-20T21:18:58.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:18:58.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:18:58.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:18:58.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror_parent test_parent 2026-02-20T21:18:58.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:58.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:18:58.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:18:58.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror_parent/test_parent 2026-02-20T21:18:58.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 5 2026-02-20T21:18:58.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror_parent mirror_parent test_parent 2026-02-20T21:18:58.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:18:58.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:58.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror_parent 2026-02-20T21:18:58.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror_parent 2026-02-20T21:18:58.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:18:58.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror_parent-test_parent.status 2026-02-20T21:18:58.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror_parent-test_parent.status 2026-02-20T21:18:58.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:18:58.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror_parent-test_parent.status 2026-02-20T21:18:58.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror_parent-test_parent.status 2026-02-20T21:18:58.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror_parent-test_parent.status 2026-02-20T21:18:58.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:18:58.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_parent-test_parent.status 2026-02-20T21:18:58.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror_parent test_parent /tmp/tmp.2FjbyoErqN/cluster2-mirror_parent-test_parent.status 2026-02-20T21:18:58.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:18:58.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:18:58.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:18:58.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror_parent-test_parent.status 2026-02-20T21:18:58.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror_parent/test_parent --format xml 2026-02-20T21:18:58.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:18:58.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:18:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=5 2026-02-20T21:18:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:18:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:18:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:18:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror_parent test_parent /tmp/tmp.2FjbyoErqN/cluster1-mirror_parent-test_parent.status 2026-02-20T21:18:58.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:58.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:18:58.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:18:58.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_parent-test_parent.status 2026-02-20T21:18:58.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror_parent/test_parent --format xml 2026-02-20T21:18:58.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:18:58.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:18:58.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=3 2026-02-20T21:18:58.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 5 = 3 2026-02-20T21:18:58.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:18:58.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.4 2026-02-20T21:18:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror_parent test_parent /tmp/tmp.2FjbyoErqN/cluster1-mirror_parent-test_parent.status 2026-02-20T21:18:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:18:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:18:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_parent-test_parent.status 2026-02-20T21:18:58.815 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror_parent/test_parent --format xml 2026-02-20T21:18:58.815 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:18:58.845 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:18:58.847 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=3 2026-02-20T21:18:58.847 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 5 = 3 2026-02-20T21:18:58.847 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:18:58.847 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.8 2026-02-20T21:18:59.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror_parent test_parent /tmp/tmp.2FjbyoErqN/cluster1-mirror_parent-test_parent.status 2026-02-20T21:18:59.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:18:59.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:18:59.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:18:59.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_parent-test_parent.status 2026-02-20T21:18:59.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror_parent/test_parent --format xml 2026-02-20T21:18:59.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:18:59.680 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:18:59.681 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:18:59.682 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:-:1.1: Document is empty 2026-02-20T21:18:59.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 2026-02-20T21:18:59.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:^ 2026-02-20T21:18:59.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id= 2026-02-20T21:18:59.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 5 = '' 2026-02-20T21:18:59.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:18:59.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1.6 2026-02-20T21:19:01.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror_parent test_parent /tmp/tmp.2FjbyoErqN/cluster1-mirror_parent-test_parent.status 2026-02-20T21:19:01.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:01.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:01.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:01.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_parent-test_parent.status 2026-02-20T21:19:01.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror_parent/test_parent --format xml 2026-02-20T21:19:01.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:19:01.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:19:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=5 2026-02-20T21:19:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 5 = 5 2026-02-20T21:19:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror_parent test_parent 2026-02-20T21:19:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:19:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror_parent test_parent up+replaying 2026-02-20T21:19:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:19:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:19:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:19:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:19:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:19:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror_parent test_parent up+replaying '' '' 2026-02-20T21:19:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:19:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:19:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:19:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:19:02.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:19:02.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror_parent/test_parent 2026-02-20T21:19:02.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_parent: 2026-02-20T21:19:02.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 0dc6cc8f-9aed-4f34-9291-5356f16c50eb 2026-02-20T21:19:02.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:19:02.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622334,"remote_snapshot_timestamp":1771622334,"replay_state":"idle"} 2026-02-20T21:19:02.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:19:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:18:58 2026-02-20T21:19:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:19:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:19:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:19:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:19:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:19:02' 2026-02-20T21:19:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:19:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:19:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:19:02.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622334,"remote_snapshot_timestamp":1771622334,"replay_state":"idle"} 2026-02-20T21:19:02.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:19:02.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:19:02.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:19:02.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:19:02.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:19:02.353 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:19:02.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:19:02.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:19:02.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:58' 2026-02-20T21:19:02.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror_parent test_parent up+replaying '2026-02-20 21:18:58' 2026-02-20T21:19:02.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:02.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:02.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:02.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:19:02.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:18:58' 2026-02-20T21:19:02.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:19:02.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:19:02.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror_parent --verbose --format xml 2026-02-20T21:19:02.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK144044420mirror.0trial19920.2.0-677-gf7870454trueOK44114430mirror.1trial19920.2.0-677-gf7870454falseOK44174436mirror.2trial19920.2.0-677-gf7870454falseOK44274439mirror.3trial19920.2.0-677-gf7870454falseOKtest_parent0dc6cc8f-9aed-4f34-9291-5356f16c50ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622334,"remote_snapshot_timestamp":1771622334,"replay_state":"idle"}44044420mirror.0trial1992026-02-20 21:18:58cluster2-DNEe15f1937-1d71-4e9c-89bd-d4d413323055up+stoppedlocal image is primary2026-02-20 21:19:02' 2026-02-20T21:19:02.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:19:02.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_parent'\'']/last_update' 2026-02-20T21:19:02.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:18:58' 2026-02-20T21:19:02.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_parent'\'']/state' 2026-02-20T21:19:02.389 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:19:02.389 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:19:02.389 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:19:02.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:19:02.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:02.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:02.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror_parent mirror_parent test_parent 2026-02-20T21:19:02.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:19:02.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:19:02.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:02.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror_parent 2026-02-20T21:19:02.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror_parent 2026-02-20T21:19:02.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:02.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror_parent-test_parent.export 2026-02-20T21:19:02.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror_parent-test_parent.export 2026-02-20T21:19:02.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:19:02.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror_parent-test_parent.export 2026-02-20T21:19:02.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror_parent-test_parent.export 2026-02-20T21:19:02.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror_parent-test_parent.export 2026-02-20T21:19:02.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:19:02.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror_parent-test_parent.export 2026-02-20T21:19:02.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror_parent-test_parent.export /tmp/tmp.2FjbyoErqN/cluster1-mirror_parent-test_parent.export 2026-02-20T21:19:02.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror_parent/test_parent /tmp/tmp.2FjbyoErqN/cluster2-mirror_parent-test_parent.export 2026-02-20T21:19:02.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:19:02.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror_parent/test_parent /tmp/tmp.2FjbyoErqN/cluster1-mirror_parent-test_parent.export 2026-02-20T21:19:02.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:19:02.568 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror_parent-test_parent.export /tmp/tmp.2FjbyoErqN/cluster1-mirror_parent-test_parent.export 2026-02-20T21:19:02.614 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror_parent-test_parent.export /tmp/tmp.2FjbyoErqN/cluster1-mirror_parent-test_parent.export 2026-02-20T21:19:02.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:02.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test_clone 2026-02-20T21:19:02.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:02.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:02.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone 2026-02-20T21:19:02.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test_clone started 2026-02-20T21:19:02.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:02.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:02.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone 2026-02-20T21:19:02.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:19:02.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:19:02.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:19:02.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:19:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test_clone started 2026-02-20T21:19:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone 2026-02-20T21:19:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:19:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:19:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:19:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_clone 2026-02-20T21:19:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:19:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:19:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:19:03.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:19:03.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:19:03.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:19:03.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:19:03.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:19:03.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:19:03.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:19:03.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone 2026-02-20T21:19:03.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:19:03.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:03.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:19:03.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:19:03.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:19:03.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:19:03.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:19:03.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:19:03.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:19:03.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:19:03.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:19:03.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:03.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:19:03.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:19:03.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:19:03.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:19:03.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:19:03.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:19:03.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:19:03.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:19:03.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:19:03.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:19:03.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:03.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:19:03.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:19:03.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:19:03.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:19:03.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:19:03.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:19:03.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:19:03.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:19:03.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:19:03.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:19:03.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone 2026-02-20T21:19:03.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_clone'] 2026-02-20T21:19:03.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:19:03.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:19:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:19:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone 2026-02-20T21:19:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:19:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:19:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:19:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:19:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:19:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:19:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:19:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:19:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:19:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:19:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:19:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:19:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:19:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:19:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:19:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:19:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:19:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:19:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:19:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:19:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:19:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:19:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:19:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:19:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:19:03.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:19:03.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:19:03.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:19:03.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:19:03.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:19:03.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone 2026-02-20T21:19:03.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:19:03.762 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:19:03.762 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:19:03.762 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:19:03.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:19:03.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:19:03.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:19:03.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test_clone 2026-02-20T21:19:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:19:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:19:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:19:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone 2026-02-20T21:19:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:19:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:19:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test_clone 2026-02-20T21:19:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone 2026-02-20T21:19:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_clone 2026-02-20T21:19:04.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 154 2026-02-20T21:19:04.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test_clone 2026-02-20T21:19:04.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:19:04.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:04.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:19:04.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:19:04.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone 2026-02-20T21:19:04.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test_clone.status 2026-02-20T21:19:04.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test_clone.status 2026-02-20T21:19:04.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:19:04.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_clone.status 2026-02-20T21:19:04.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test_clone.status 2026-02-20T21:19:04.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test_clone.status 2026-02-20T21:19:04.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:19:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_clone.status 2026-02-20T21:19:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_clone /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_clone.status 2026-02-20T21:19:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone 2026-02-20T21:19:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_clone.status 2026-02-20T21:19:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_clone --format xml 2026-02-20T21:19:04.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:19:04.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:19:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=154 2026-02-20T21:19:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:19:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:19:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:19:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_clone /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_clone.status 2026-02-20T21:19:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone 2026-02-20T21:19:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_clone.status 2026-02-20T21:19:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_clone --format xml 2026-02-20T21:19:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:19:04.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:19:04.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=153 2026-02-20T21:19:04.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 154 = 153 2026-02-20T21:19:04.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:19:04.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.4 2026-02-20T21:19:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_clone /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_clone.status 2026-02-20T21:19:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone 2026-02-20T21:19:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_clone.status 2026-02-20T21:19:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_clone --format xml 2026-02-20T21:19:04.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:19:04.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:19:04.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=153 2026-02-20T21:19:04.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 154 = 153 2026-02-20T21:19:04.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:19:04.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.8 2026-02-20T21:19:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_clone /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_clone.status 2026-02-20T21:19:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone 2026-02-20T21:19:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_clone.status 2026-02-20T21:19:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_clone --format xml 2026-02-20T21:19:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:19:05.702 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:19:05.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=154 2026-02-20T21:19:05.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 154 = 154 2026-02-20T21:19:05.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:05.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_clone 2026-02-20T21:19:05.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:05.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:05.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone 2026-02-20T21:19:05.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:19:05.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_clone up+replaying 2026-02-20T21:19:05.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:05.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:05.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone 2026-02-20T21:19:05.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:19:05.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:19:05.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:19:05.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:19:05.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:19:06.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_clone up+replaying '' '' 2026-02-20T21:19:06.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:06.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:06.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone 2026-02-20T21:19:06.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:19:06.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:19:06.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:19:06.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:19:06.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:19:06.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_clone 2026-02-20T21:19:06.742 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_clone: 2026-02-20T21:19:06.742 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 3027f03e-b1b6-46ff-8902-5ee7692d9b2e 2026-02-20T21:19:06.742 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:19:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622336,"replay_state":"idle"} 2026-02-20T21:19:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:19:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:19:03 2026-02-20T21:19:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:19:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:19:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: down+unknown 2026-02-20T21:19:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: status not found 2026-02-20T21:19:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: ' 2026-02-20T21:19:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:19:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:19:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:19:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622336,"replay_state":"idle"} 2026-02-20T21:19:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:19:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:19:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:19:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:19:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:19:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:19:06.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:19:06.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:19:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:19:03' 2026-02-20T21:19:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_clone up+replaying '2026-02-20 21:19:03' 2026-02-20T21:19:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone 2026-02-20T21:19:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:19:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:19:03' 2026-02-20T21:19:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:19:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:19:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:19:06.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING1144044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:18:58cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:19:06test_clone3027f03e-b1b6-46ff-8902-5ee7692d9b2eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622336,"replay_state":"idle"}44114415mirror.1trial1992026-02-20 21:19:03cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8down+unknownstatus not found' 2026-02-20T21:19:06.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:19:06.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_clone'\'']/last_update' 2026-02-20T21:19:06.786 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:19:03' 2026-02-20T21:19:06.786 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_clone'\'']/state' 2026-02-20T21:19:06.788 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:19:06.788 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:19:06.788 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:19:06.789 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:19:06.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:06.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:06.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror mirror test_clone 2026-02-20T21:19:06.791 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:19:06.791 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:19:06.791 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:06.791 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:19:06.791 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:19:06.791 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone 2026-02-20T21:19:06.791 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test_clone.export 2026-02-20T21:19:06.791 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test_clone.export 2026-02-20T21:19:06.791 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:19:06.792 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_clone.export 2026-02-20T21:19:06.792 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test_clone.export 2026-02-20T21:19:06.792 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test_clone.export 2026-02-20T21:19:06.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:19:06.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_clone.export 2026-02-20T21:19:06.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_clone.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_clone.export 2026-02-20T21:19:06.794 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/test_clone /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_clone.export 2026-02-20T21:19:06.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:19:06.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/test_clone /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_clone.export 2026-02-20T21:19:06.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:19:06.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_clone.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_clone.export 2026-02-20T21:19:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_clone.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_clone.export 2026-02-20T21:19:07.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:07.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster2 mirror test_clone 2026-02-20T21:19:07.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:07.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:07.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone 2026-02-20T21:19:07.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:19:07.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:19:07.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster2 mirror test_clone 2026-02-20T21:19:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone 2026-02-20T21:19:07.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_clone 2026-02-20T21:19:07.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rm mirror/test_clone 2026-02-20T21:19:09.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:19:09.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:09.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog ' - clone v1' 2026-02-20T21:19:09.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:19:09.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:19:09.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:19:09 - clone v1 2026-02-20T21:19:09.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:19:09 - clone v1 2026-02-20T21:19:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ clone_image_and_enable_mirror cluster1 mirror_parent test_parent snap mirror test_clone1 snapshot 2026-02-20T21:19:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_pool=mirror_parent 2026-02-20T21:19:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_image=test_parent 2026-02-20T21:19:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_snap=snap 2026-02-20T21:19:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local clone_pool=mirror 2026-02-20T21:19:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local clone_image=test_clone1 2026-02-20T21:19:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 6 2026-02-20T21:19:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:19:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:19:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:19:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ clone_image cluster1 mirror_parent test_parent snap mirror test_clone1 2026-02-20T21:19:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_pool=mirror_parent 2026-02-20T21:19:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_image=test_parent 2026-02-20T21:19:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_snap=snap 2026-02-20T21:19:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local clone_pool=mirror 2026-02-20T21:19:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local clone_image=test_clone1 2026-02-20T21:19:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 6 2026-02-20T21:19:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 clone mirror_parent/test_parent@snap mirror/test_clone1 --image-feature layering,exclusive-lock,object-map,fast-diff 2026-02-20T21:19:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:19:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:19:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster1 mirror test_clone1 snapshot 2026-02-20T21:19:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone1 2026-02-20T21:19:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:19:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/test_clone1 snapshot 2026-02-20T21:19:10.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:19:10.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 info mirror/test_clone1 2026-02-20T21:19:10.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test_clone1': 2026-02-20T21:19:10.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:19:10.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:19:10.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:19:10.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 157f9c1be8c9 2026-02-20T21:19:10.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.157f9c1be8c9 2026-02-20T21:19:10.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:19:10.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:19:10.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:19:10.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:19:10.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:19:09 2026 2026-02-20T21:19:10.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:19:09 2026 2026-02-20T21:19:10.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:19:09 2026 2026-02-20T21:19:10.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: parent: mirror_parent/test_parent@snap 2026-02-20T21:19:10.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: overlap: 128 MiB 2026-02-20T21:19:10.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:19:10.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:19:10.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 3fc0a683-3e6f-4880-b0cb-74a9a0fa63a4 2026-02-20T21:19:10.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:19:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ clone_image_and_enable_mirror cluster2 mirror_parent test_parent snap mirror test_clone_v1 snapshot --rbd-default-clone-format 1 2026-02-20T21:19:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_pool=mirror_parent 2026-02-20T21:19:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_image=test_parent 2026-02-20T21:19:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_snap=snap 2026-02-20T21:19:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local clone_pool=mirror 2026-02-20T21:19:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local clone_image=test_clone_v1 2026-02-20T21:19:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 6 2026-02-20T21:19:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:19:10.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:19:10.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:19:10.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ clone_image cluster2 mirror_parent test_parent snap mirror test_clone_v1 --rbd-default-clone-format 1 2026-02-20T21:19:10.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:10.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_pool=mirror_parent 2026-02-20T21:19:10.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_image=test_parent 2026-02-20T21:19:10.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_snap=snap 2026-02-20T21:19:10.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local clone_pool=mirror 2026-02-20T21:19:10.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local clone_image=test_clone_v1 2026-02-20T21:19:10.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 6 2026-02-20T21:19:10.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 clone mirror_parent/test_parent@snap mirror/test_clone_v1 --image-feature layering,exclusive-lock,object-map,fast-diff --rbd-default-clone-format 1 2026-02-20T21:19:10.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:19:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:19:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror test_clone_v1 snapshot 2026-02-20T21:19:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone_v1 2026-02-20T21:19:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:19:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_clone_v1 snapshot 2026-02-20T21:19:11.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:19:11.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/test_clone_v1 2026-02-20T21:19:11.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test_clone_v1': 2026-02-20T21:19:11.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:19:11.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:19:11.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:19:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 15b68d2c9b8c 2026-02-20T21:19:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.15b68d2c9b8c 2026-02-20T21:19:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:19:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:19:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:19:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:19:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:19:10 2026 2026-02-20T21:19:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:19:10 2026 2026-02-20T21:19:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:19:10 2026 2026-02-20T21:19:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: parent: mirror_parent/test_parent@snap 2026-02-20T21:19:11.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: overlap: 128 MiB 2026-02-20T21:19:11.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:19:11.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:19:11.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: da10fb76-cd00-4d74-8f59-2214080a493b 2026-02-20T21:19:11.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:19:11.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_clone_format cluster2 mirror test_clone_v1 2026-02-20T21:19:11.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:19:11.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:19:11.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test_clone_v1 2026-02-20T21:19:11.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 info mirror/test_clone_v1 2026-02-20T21:19:11.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ awk 'BEGIN { 2026-02-20T21:19:11.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: format = 1 2026-02-20T21:19:11.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: } 2026-02-20T21:19:11.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: $1 == "parent:" { 2026-02-20T21:19:11.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: parent = $2 2026-02-20T21:19:11.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: } 2026-02-20T21:19:11.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: /op_features: .*clone-child/ { 2026-02-20T21:19:11.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: format = 2 2026-02-20T21:19:11.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: } 2026-02-20T21:19:11.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: END { 2026-02-20T21:19:11.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: if (!parent) exit 1 2026-02-20T21:19:11.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: print format 2026-02-20T21:19:11.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: }' 2026-02-20T21:19:11.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 1 = 1 2026-02-20T21:19:11.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test_clone_v1 2026-02-20T21:19:11.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:11.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:11.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone_v1 2026-02-20T21:19:11.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test_clone_v1 started 2026-02-20T21:19:11.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:11.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:11.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone_v1 2026-02-20T21:19:11.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:19:11.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:19:11.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:19:11.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:19:12.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test_clone_v1 started 2026-02-20T21:19:12.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:12.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:12.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone_v1 2026-02-20T21:19:12.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:19:12.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:19:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:19:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_clone_v1 2026-02-20T21:19:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:19:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:19:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:19:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:19:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:19:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:19:12.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:19:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:19:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:19:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:19:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v1 2026-02-20T21:19:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:19:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:19:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:19:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:19:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:19:12.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:19:12.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:19:12.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:19:12.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:19:12.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:19:12.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:12.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:19:12.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:19:12.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:19:12.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:19:12.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:19:12.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:19:12.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:19:12.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:19:12.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:19:12.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:19:12.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:12.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:19:12.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:19:12.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:19:12.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:19:12.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:19:12.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:19:12.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:19:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:19:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:19:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:19:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v1 2026-02-20T21:19:12.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-20T21:19:12.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:19:12.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:19:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:19:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone_v1 2026-02-20T21:19:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:19:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:19:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:19:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:19:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:19:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:19:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:19:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:19:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:19:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:19:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:19:12.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:19:12.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:19:12.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:19:12.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:19:12.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:19:12.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:19:12.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:19:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:19:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:19:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:19:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:19:12.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:19:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:19:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:19:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:19:12.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:19:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:19:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:19:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:19:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone_v1 2026-02-20T21:19:12.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-20T21:19:12.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:19:12.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:19:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:19:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_clone_v1 2026-02-20T21:19:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:19:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:19:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:19:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:19:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:19:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:19:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:19:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:19:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:19:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:19:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:19:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:19:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:19:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:19:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:19:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:19:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:19:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:19:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:19:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:19:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:19:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:19:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:19:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:19:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:19:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:19:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:19:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:19:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:19:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:19:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_clone_v1 2026-02-20T21:19:12.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-20T21:19:12.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:19:12.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:19:12.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:19:12.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_clone_v1 2026-02-20T21:19:12.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:19:12.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:19:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:19:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:19:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:19:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:19:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:19:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:19:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:19:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:19:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:19:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:19:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:19:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:19:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:19:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:19:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:19:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:19:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:19:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:19:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:19:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:19:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:19:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:19:12.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:19:12.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:19:12.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:19:12.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:19:12.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:19:12.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:19:12.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_clone_v1 2026-02-20T21:19:12.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-20T21:19:12.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:19:12.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:19:12.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:19:12.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 1 2026-02-20T21:19:13.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:19:13.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:19:13.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:19:13.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v1 2026-02-20T21:19:13.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:19:13.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:13.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:19:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:19:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:19:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:19:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:19:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:19:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:19:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:19:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:19:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:19:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:19:13.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:19:13.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:19:13.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:19:13.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:19:13.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:19:13.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:19:13.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:19:13.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:19:13.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:13.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:19:13.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:19:13.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:19:13.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:19:13.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:19:13.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:19:13.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:19:13.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:19:13.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:19:13.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:19:13.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v1 2026-02-20T21:19:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-20T21:19:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:19:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:19:13.507 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:19:13.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone_v1 2026-02-20T21:19:13.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:19:13.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:13.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:19:13.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:19:13.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:19:13.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:19:13.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:19:13.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:19:13.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:19:13.508 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:19:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:19:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:19:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:19:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:19:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:19:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:19:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:19:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:19:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:19:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:19:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:19:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:13.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:19:13.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:19:13.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:19:13.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:19:13.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:19:13.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:19:13.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:19:13.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:19:13.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:19:13.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:19:13.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone_v1 2026-02-20T21:19:13.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-20T21:19:13.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:19:13.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:19:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:19:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_clone_v1 2026-02-20T21:19:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:19:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:19:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:19:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:19:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:19:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:19:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:19:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:19:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:19:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:19:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:19:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:19:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:19:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:19:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:19:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:19:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:19:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:19:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:19:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:19:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:19:13.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:19:13.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:19:13.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:19:13.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:19:13.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:19:13.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:19:13.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:19:13.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:19:13.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:19:13.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_clone_v1 2026-02-20T21:19:13.619 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:19:13.619 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:19:13.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:19:13.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:19:13.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:19:13.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:19:13.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:19:13.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_clone_format cluster1 mirror test_clone_v1 2026-02-20T21:19:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:19:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:19:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test_clone_v1 2026-02-20T21:19:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 info mirror/test_clone_v1 2026-02-20T21:19:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ awk 'BEGIN { 2026-02-20T21:19:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: format = 1 2026-02-20T21:19:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: } 2026-02-20T21:19:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: $1 == "parent:" { 2026-02-20T21:19:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: parent = $2 2026-02-20T21:19:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: } 2026-02-20T21:19:13.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: /op_features: .*clone-child/ { 2026-02-20T21:19:13.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: format = 2 2026-02-20T21:19:13.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: } 2026-02-20T21:19:13.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: END { 2026-02-20T21:19:13.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: if (!parent) exit 1 2026-02-20T21:19:13.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: print format 2026-02-20T21:19:13.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: }' 2026-02-20T21:19:13.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 1 = 1 2026-02-20T21:19:13.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster2 mirror test_clone_v1 2026-02-20T21:19:13.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:13.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:13.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone_v1 2026-02-20T21:19:13.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:19:13.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:19:13.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster2 mirror test_clone_v1 2026-02-20T21:19:13.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:13.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:13.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone_v1 2026-02-20T21:19:13.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_clone_v1 2026-02-20T21:19:13.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rm mirror/test_clone_v1 2026-02-20T21:19:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:19:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster1 mirror test_clone1 2026-02-20T21:19:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone1 2026-02-20T21:19:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:19:14.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:19:14.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster1 mirror test_clone1 2026-02-20T21:19:14.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:14.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:14.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone1 2026-02-20T21:19:14.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_clone1 2026-02-20T21:19:14.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 rm mirror/test_clone1 2026-02-20T21:19:15.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:19:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ unprotect_snapshot_retry cluster2 mirror_parent test_parent snap 2026-02-20T21:19:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap 2026-02-20T21:19:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:19:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:19:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ unprotect_snapshot cluster2 mirror_parent test_parent snap 2026-02-20T21:19:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap 2026-02-20T21:19:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap unprotect mirror_parent/test_parent@snap 2026-02-20T21:19:15.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: unprotecting snap failed: (16) Device or resource busy 2026-02-20T21:19:15.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:19:15.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:19:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ unprotect_snapshot cluster2 mirror_parent test_parent snap 2026-02-20T21:19:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap 2026-02-20T21:19:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap unprotect mirror_parent/test_parent@snap 2026-02-20T21:19:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_snapshot cluster2 mirror_parent test_parent snap 2026-02-20T21:19:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap 2026-02-20T21:19:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rm mirror_parent/test_parent@snap 2026-02-20T21:19:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing snap: 100% complete...done. 2026-02-20T21:19:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog ' - clone v2' 2026-02-20T21:19:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:19:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:19:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:19:16 - clone v2 2026-02-20T21:19:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:19:16 - clone v2 2026-02-20T21:19:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ parent_snap=snap_v2 2026-02-20T21:19:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_snapshot cluster2 mirror_parent test_parent snap_v2 2026-02-20T21:19:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_v2 2026-02-20T21:19:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap create mirror_parent/test_parent@snap_v2 2026-02-20T21:19:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-20T21:19:17.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:19:17.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror_parent test_parent 2026-02-20T21:19:17.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:17.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:17.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:17.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror_parent/test_parent 2026-02-20T21:19:19.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 7 2026-02-20T21:19:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ clone_image_and_enable_mirror cluster2 mirror_parent test_parent snap_v2 mirror test_clone_v2 snapshot --rbd-default-clone-format 2 2026-02-20T21:19:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_pool=mirror_parent 2026-02-20T21:19:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_image=test_parent 2026-02-20T21:19:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_snap=snap_v2 2026-02-20T21:19:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local clone_pool=mirror 2026-02-20T21:19:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local clone_image=test_clone_v2 2026-02-20T21:19:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 6 2026-02-20T21:19:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:19:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:19:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:19:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ clone_image cluster2 mirror_parent test_parent snap_v2 mirror test_clone_v2 --rbd-default-clone-format 2 2026-02-20T21:19:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_pool=mirror_parent 2026-02-20T21:19:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_image=test_parent 2026-02-20T21:19:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_snap=snap_v2 2026-02-20T21:19:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local clone_pool=mirror 2026-02-20T21:19:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local clone_image=test_clone_v2 2026-02-20T21:19:19.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 6 2026-02-20T21:19:19.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 clone mirror_parent/test_parent@snap_v2 mirror/test_clone_v2 --image-feature layering,exclusive-lock,object-map,fast-diff --rbd-default-clone-format 2 2026-02-20T21:19:19.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:19:19.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:19.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:19:19.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:19.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror test_clone_v2 snapshot 2026-02-20T21:19:19.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:19.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:19.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone_v2 2026-02-20T21:19:19.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:19:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_clone_v2 snapshot 2026-02-20T21:19:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:19:20.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/test_clone_v2 2026-02-20T21:19:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test_clone_v2': 2026-02-20T21:19:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:19:20.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:19:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:19:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 15d725721d94 2026-02-20T21:19:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.15d725721d94 2026-02-20T21:19:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:19:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff, operations 2026-02-20T21:19:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: clone-child 2026-02-20T21:19:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:19:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:19:19 2026 2026-02-20T21:19:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:19:19 2026 2026-02-20T21:19:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:19:19 2026 2026-02-20T21:19:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: parent: mirror_parent/test_parent@snap_v2 2026-02-20T21:19:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: overlap: 128 MiB 2026-02-20T21:19:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:19:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:19:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 59bc031f-d736-4abd-8795-79857fa78761 2026-02-20T21:19:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:19:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_clone_format cluster2 mirror test_clone_v2 2026-02-20T21:19:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:19:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:19:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test_clone_v2 2026-02-20T21:19:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 info mirror/test_clone_v2 2026-02-20T21:19:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ awk 'BEGIN { 2026-02-20T21:19:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: format = 1 2026-02-20T21:19:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: } 2026-02-20T21:19:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: $1 == "parent:" { 2026-02-20T21:19:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: parent = $2 2026-02-20T21:19:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: } 2026-02-20T21:19:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: /op_features: .*clone-child/ { 2026-02-20T21:19:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: format = 2 2026-02-20T21:19:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: } 2026-02-20T21:19:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: END { 2026-02-20T21:19:20.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: if (!parent) exit 1 2026-02-20T21:19:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: print format 2026-02-20T21:19:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: }' 2026-02-20T21:19:20.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 2 = 2 2026-02-20T21:19:20.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test_clone_v2 2026-02-20T21:19:20.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:20.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:20.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone_v2 2026-02-20T21:19:20.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test_clone_v2 started 2026-02-20T21:19:20.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:20.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:20.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone_v2 2026-02-20T21:19:20.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:19:20.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:19:20.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:19:20.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:19:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test_clone_v2 started 2026-02-20T21:19:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone_v2 2026-02-20T21:19:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:19:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:19:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:19:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_clone_v2 2026-02-20T21:19:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:19:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:19:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:19:21.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:19:21.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:19:21.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:19:21.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:19:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:19:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:19:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:19:21.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v2 2026-02-20T21:19:21.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:19:21.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:21.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:19:21.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:19:21.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:19:21.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:19:21.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:19:21.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:19:21.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:19:21.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:19:21.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:19:21.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:19:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:19:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:19:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:19:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:19:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:19:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:19:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:19:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:19:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:19:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:19:21.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:19:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:19:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:19:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:19:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:19:21.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:19:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:19:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:19:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:19:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v2 2026-02-20T21:19:21.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-20T21:19:21.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:19:21.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:19:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:19:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone_v2 2026-02-20T21:19:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:19:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:19:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:19:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:19:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:19:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:19:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:19:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:19:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:19:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:19:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:19:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:19:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:19:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:19:21.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:19:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:19:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:19:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:19:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:19:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:19:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:19:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:19:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:19:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:19:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:19:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:19:21.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:19:21.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:19:21.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:19:21.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:19:21.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone_v2 2026-02-20T21:19:21.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-20T21:19:21.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:19:21.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:19:21.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:19:21.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_clone_v2 2026-02-20T21:19:21.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:19:21.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:21.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:19:21.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:19:21.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:19:21.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:19:21.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:19:21.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:19:21.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:19:21.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:19:21.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:19:21.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:21.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:19:21.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:19:21.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:19:21.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:19:21.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:19:21.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:19:21.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:19:21.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:19:21.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:19:21.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:19:21.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:21.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:19:21.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:19:21.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:19:21.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:19:21.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:19:21.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:19:21.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:19:21.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:19:21.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:19:21.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:19:21.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_clone_v2 2026-02-20T21:19:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-20T21:19:21.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:19:21.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:19:21.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:19:21.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_clone_v2 2026-02-20T21:19:21.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:19:21.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:21.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:19:21.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:19:21.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:19:21.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:19:21.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:19:21.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:19:21.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:19:21.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:19:21.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:19:21.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:21.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:19:21.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:19:21.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:19:21.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:19:21.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:19:21.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:19:21.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:19:21.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:19:21.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:19:21.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:19:21.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:19:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:19:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:19:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:19:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:19:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:19:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:19:21.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:19:21.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:19:21.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:19:21.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_clone_v2 2026-02-20T21:19:21.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-20T21:19:21.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:19:21.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:19:21.443 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:19:21.443 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 1 2026-02-20T21:19:22.443 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:19:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:19:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:19:22.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v2 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:19:22.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:19:22.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:19:22.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:19:22.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:19:22.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v2 2026-02-20T21:19:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-20T21:19:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:19:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:19:22.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:19:22.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone_v2 2026-02-20T21:19:22.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:19:22.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:22.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:19:22.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:19:22.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:19:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:19:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:19:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:19:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:19:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:19:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:19:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:19:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:19:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:19:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:19:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:19:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:19:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:19:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:19:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:19:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:19:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:19:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:19:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:19:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:19:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:19:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:19:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:19:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:19:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:19:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:19:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone_v2 2026-02-20T21:19:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:19:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:19:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:19:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:19:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:19:22.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:19:22.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:19:22.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:22.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_clone_format cluster1 mirror test_clone_v2 2026-02-20T21:19:22.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:19:22.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:19:22.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test_clone_v2 2026-02-20T21:19:22.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 info mirror/test_clone_v2 2026-02-20T21:19:22.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ awk 'BEGIN { 2026-02-20T21:19:22.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: format = 1 2026-02-20T21:19:22.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: } 2026-02-20T21:19:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: $1 == "parent:" { 2026-02-20T21:19:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: parent = $2 2026-02-20T21:19:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: } 2026-02-20T21:19:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: /op_features: .*clone-child/ { 2026-02-20T21:19:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: format = 2 2026-02-20T21:19:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: } 2026-02-20T21:19:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: END { 2026-02-20T21:19:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: if (!parent) exit 1 2026-02-20T21:19:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: print format 2026-02-20T21:19:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: }' 2026-02-20T21:19:22.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 2 = 2 2026-02-20T21:19:22.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_snapshot cluster2 mirror_parent test_parent snap_v2 2026-02-20T21:19:22.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:22.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:22.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:22.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_v2 2026-02-20T21:19:22.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rm mirror_parent/test_parent@snap_v2 2026-02-20T21:19:22.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing snap: 100% complete...done. 2026-02-20T21:19:22.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:19:22.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror_parent test_parent 2026-02-20T21:19:22.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:22.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:22.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:22.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror_parent/test_parent 2026-02-20T21:19:23.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 8 2026-02-20T21:19:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_snap_moved_to_trash cluster2 mirror_parent test_parent snap_v2 2026-02-20T21:19:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:23.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap_name=snap_v2 2026-02-20T21:19:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap ls mirror_parent/test_parent --all 2026-02-20T21:19:23.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-20T21:19:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: 6 e6cdc43f-3389-4ed6-8cfe-d682e8d02be2 128 MiB Fri Feb 20 21:19:17 2026 trash (user snap_v2) 2026-02-20T21:19:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-20T21:19:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:19:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:19:23.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:19:24.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-20T21:19:24.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:24.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:24.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:24.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap_name=snap_v2 2026-02-20T21:19:24.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-02-20T21:19:24.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-20T21:19:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ continue 2026-02-20T21:19:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:19:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:19:26.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-20T21:19:26.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:26.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:26.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:26.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap_name=snap_v2 2026-02-20T21:19:26.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-02-20T21:19:26.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-20T21:19:26.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: 6 5ee91877-2238-4229-a62e-9dcfc85b34c8 128 MiB Fri Feb 20 21:19:19 2026 trash (user snap_v2) 2026-02-20T21:19:26.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:26.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster2 mirror test_clone_v2 2026-02-20T21:19:26.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:26.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:26.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone_v2 2026-02-20T21:19:26.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:19:26.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:19:26.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster2 mirror test_clone_v2 2026-02-20T21:19:26.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:26.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:26.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone_v2 2026-02-20T21:19:26.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_clone_v2 2026-02-20T21:19:26.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rm mirror/test_clone_v2 2026-02-20T21:19:27.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:19:27.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:27.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror test_clone_v2 deleted 2026-02-20T21:19:27.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:27.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:27.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone_v2 2026-02-20T21:19:27.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=deleted 2026-02-20T21:19:27.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:19:27.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:19:27.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:19:27.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test_clone_v2 2026-02-20T21:19:27.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:19:27.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:19:27.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test_clone_v2 2026-02-20T21:19:27.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test_clone_v2 2026-02-20T21:19:27.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:19:27.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test_clone_v2: (2) No such file or directory 2026-02-20T21:19:27.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image_id= 2026-02-20T21:19:27.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:19:27.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:19:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test_clone_v2 deleted '' 2026-02-20T21:19:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone_v2 2026-02-20T21:19:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=deleted 2026-02-20T21:19:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:19:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:19:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:19:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test_clone_v2 2026-02-20T21:19:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:19:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:19:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test_clone_v2 2026-02-20T21:19:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test_clone_v2 2026-02-20T21:19:28.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:19:28.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test_clone_v2: (2) No such file or directory 2026-02-20T21:19:28.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id= 2026-02-20T21:19:28.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:19:28.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test deleted = deleted 2026-02-20T21:19:28.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:28.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_snap_removed_from_trash cluster2 mirror_parent test_parent snap_v2 2026-02-20T21:19:28.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_snap_moved_to_trash cluster2 mirror_parent test_parent snap_v2 2026-02-20T21:19:28.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:28.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:28.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:28.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap_name=snap_v2 2026-02-20T21:19:28.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap ls mirror_parent/test_parent --all 2026-02-20T21:19:28.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-20T21:19:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:28.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snap_removed_from_trash cluster1 mirror_parent test_parent snap_v2 2026-02-20T21:19:28.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:19:28.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:19:28.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:19:29.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_snap_removed_from_trash cluster1 mirror_parent test_parent snap_v2 2026-02-20T21:19:29.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-20T21:19:29.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:29.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:29.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:29.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap_name=snap_v2 2026-02-20T21:19:29.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-02-20T21:19:29.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-20T21:19:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog ' - clone v2 non-primary' 2026-02-20T21:19:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:19:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:19:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:19:29 - clone v2 non-primary 2026-02-20T21:19:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:19:29 - clone v2 non-primary 2026-02-20T21:19:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_snapshot cluster2 mirror_parent test_parent snap_v2 2026-02-20T21:19:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_v2 2026-02-20T21:19:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap create mirror_parent/test_parent@snap_v2 2026-02-20T21:19:29.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-20T21:19:29.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:19:29.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror_parent test_parent 2026-02-20T21:19:29.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:29.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:29.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:29.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror_parent/test_parent 2026-02-20T21:19:32.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 10 2026-02-20T21:19:32.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snap_present cluster1 mirror_parent test_parent snap_v2 2026-02-20T21:19:32.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:32.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:32.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:32.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap_name=snap_v2 2026-02-20T21:19:32.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:19:32.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:19:32.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:19:33.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 info mirror_parent/test_parent@snap_v2 2026-02-20T21:19:33.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test_parent': 2026-02-20T21:19:33.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:19:33.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:19:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 2 2026-02-20T21:19:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 1144304168d3 2026-02-20T21:19:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.1144304168d3 2026-02-20T21:19:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:19:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:19:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:19:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:19:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:18:56 2026 2026-02-20T21:19:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:18:56 2026 2026-02-20T21:19:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:18:56 2026 2026-02-20T21:19:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: protected: False 2026-02-20T21:19:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:19:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:19:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 0dc6cc8f-9aed-4f34-9291-5356f16c50eb 2026-02-20T21:19:33.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: false 2026-02-20T21:19:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ clone_image_and_enable_mirror cluster1 mirror_parent test_parent snap_v2 mirror test_clone_v2 snapshot --rbd-default-clone-format 2 2026-02-20T21:19:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_pool=mirror_parent 2026-02-20T21:19:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_image=test_parent 2026-02-20T21:19:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_snap=snap_v2 2026-02-20T21:19:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local clone_pool=mirror 2026-02-20T21:19:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local clone_image=test_clone_v2 2026-02-20T21:19:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 6 2026-02-20T21:19:33.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:19:33.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:19:33.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:19:33.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ clone_image cluster1 mirror_parent test_parent snap_v2 mirror test_clone_v2 --rbd-default-clone-format 2 2026-02-20T21:19:33.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:33.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_pool=mirror_parent 2026-02-20T21:19:33.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_image=test_parent 2026-02-20T21:19:33.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local parent_snap=snap_v2 2026-02-20T21:19:33.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local clone_pool=mirror 2026-02-20T21:19:33.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local clone_image=test_clone_v2 2026-02-20T21:19:33.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 6 2026-02-20T21:19:33.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 clone mirror_parent/test_parent@snap_v2 mirror/test_clone_v2 --image-feature layering,exclusive-lock,object-map,fast-diff --rbd-default-clone-format 2 2026-02-20T21:19:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:19:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:19:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster1 mirror test_clone_v2 snapshot 2026-02-20T21:19:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone_v2 2026-02-20T21:19:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:19:34.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/test_clone_v2 snapshot 2026-02-20T21:19:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:19:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 info mirror/test_clone_v2 2026-02-20T21:19:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test_clone_v2': 2026-02-20T21:19:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:19:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:19:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:19:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 15a29e258e9 2026-02-20T21:19:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.15a29e258e9 2026-02-20T21:19:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:19:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff, operations 2026-02-20T21:19:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: clone-child 2026-02-20T21:19:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:19:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:19:34 2026 2026-02-20T21:19:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:19:34 2026 2026-02-20T21:19:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:19:34 2026 2026-02-20T21:19:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: parent: mirror_parent/test_parent@snap_v2 2026-02-20T21:19:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: overlap: 128 MiB 2026-02-20T21:19:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:19:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:19:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: d35d1174-9dbe-49ac-b5da-fab1c4b4339d 2026-02-20T21:19:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:19:35.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_snapshot cluster2 mirror_parent test_parent snap_v2 2026-02-20T21:19:35.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:35.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:35.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:35.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_v2 2026-02-20T21:19:35.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rm mirror_parent/test_parent@snap_v2 2026-02-20T21:19:35.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing snap: 100% complete...done. 2026-02-20T21:19:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_snap_removed_from_trash cluster2 mirror_parent test_parent snap_v2 2026-02-20T21:19:35.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_snap_moved_to_trash cluster2 mirror_parent test_parent snap_v2 2026-02-20T21:19:35.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:35.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:35.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:35.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap_name=snap_v2 2026-02-20T21:19:35.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap ls mirror_parent/test_parent --all 2026-02-20T21:19:35.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-20T21:19:35.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:35.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:19:35.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror_parent test_parent 2026-02-20T21:19:35.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:35.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:35.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:35.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror_parent/test_parent 2026-02-20T21:19:37.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 11 2026-02-20T21:19:37.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-20T21:19:37.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:19:37.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:19:37.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:19:38.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-20T21:19:38.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:38.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:38.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:38.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap_name=snap_v2 2026-02-20T21:19:38.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-02-20T21:19:38.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-20T21:19:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: 9 a260ef87-a2b6-4b41-bed9-075ba0ebdab5 128 MiB Fri Feb 20 21:19:31 2026 trash (user snap_v2) 2026-02-20T21:19:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster1 mirror test_clone_v2 2026-02-20T21:19:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone_v2 2026-02-20T21:19:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:19:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:19:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster1 mirror test_clone_v2 2026-02-20T21:19:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_clone_v2 2026-02-20T21:19:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_clone_v2 2026-02-20T21:19:39.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 rm mirror/test_clone_v2 2026-02-20T21:19:41.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:19:41.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:41.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snap_removed_from_trash cluster1 mirror_parent test_parent snap_v2 2026-02-20T21:19:41.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:19:41.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:19:41.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:19:42.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_snap_removed_from_trash cluster1 mirror_parent test_parent snap_v2 2026-02-20T21:19:42.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-20T21:19:42.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:42.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:42.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:42.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap_name=snap_v2 2026-02-20T21:19:42.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-02-20T21:19:42.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-20T21:19:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster2 mirror_parent test_parent 2026-02-20T21:19:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:42.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:19:42.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:19:42.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster2 mirror_parent test_parent 2026-02-20T21:19:42.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:42.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror_parent 2026-02-20T21:19:42.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_parent 2026-02-20T21:19:42.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 snap purge mirror_parent/test_parent 2026-02-20T21:19:42.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rm mirror_parent/test_parent 2026-02-20T21:19:43.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:19:43.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:43.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: data pool' 2026-02-20T21:19:43.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:19:43.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:19:43.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:19:43 TEST: data pool 2026-02-20T21:19:43.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:19:43 TEST: data pool 2026-02-20T21:19:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ dp_image=test_data_pool 2026-02-20T21:19:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image_and_enable_mirror cluster2 mirror test_data_pool snapshot 128 --data-pool mirror_parent 2026-02-20T21:19:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:19:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:19:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:19:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:19:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:19:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:19:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image cluster2 mirror test_data_pool 128 --data-pool mirror_parent 2026-02-20T21:19:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:19:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:19:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:19:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=128 2026-02-20T21:19:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 128 ']' 2026-02-20T21:19:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=128 2026-02-20T21:19:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:19:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,object-map,fast-diff --data-pool mirror_parent mirror/test_data_pool 2026-02-20T21:19:43.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:19:43.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:43.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:19:43.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:43.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror test_data_pool snapshot 2026-02-20T21:19:43.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:43.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:43.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:43.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:19:43.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_data_pool snapshot 2026-02-20T21:19:44.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:19:44.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/test_data_pool 2026-02-20T21:19:44.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test_data_pool': 2026-02-20T21:19:44.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:19:44.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:19:44.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:19:44.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 160a9705d5f1 2026-02-20T21:19:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: data_pool: mirror_parent 2026-02-20T21:19:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.3.160a9705d5f1 2026-02-20T21:19:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:19:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff, data-pool 2026-02-20T21:19:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:19:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:19:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:19:43 2026 2026-02-20T21:19:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:19:43 2026 2026-02-20T21:19:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:19:43 2026 2026-02-20T21:19:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:19:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:19:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 089510a4-b64d-4031-a8a6-893deb802b04 2026-02-20T21:19:44.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:19:44.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_data_pool cluster2 mirror test_data_pool 2026-02-20T21:19:44.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:19:44.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:19:44.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test_data_pool 2026-02-20T21:19:44.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 info mirror/test_data_pool 2026-02-20T21:19:44.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ awk '$1 == "data_pool:" {print $2}' 2026-02-20T21:19:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ data_pool=mirror_parent 2026-02-20T21:19:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test mirror_parent = mirror_parent 2026-02-20T21:19:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test_data_pool 2026-02-20T21:19:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test_data_pool started 2026-02-20T21:19:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:19:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:19:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:19:44.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:19:45.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test_data_pool started 2026-02-20T21:19:45.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:45.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:45.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:19:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:19:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:19:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_data_pool 2026-02-20T21:19:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:19:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:19:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:19:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:19:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:19:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:19:45.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:19:45.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:19:45.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:19:45.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:19:45.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_data_pool 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:19:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:19:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:19:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:19:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:19:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_data_pool 2026-02-20T21:19:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-02-20T21:19:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:19:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:19:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:19:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_data_pool 2026-02-20T21:19:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:19:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:19:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:19:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:19:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:19:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:19:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:19:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:19:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:19:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:19:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:19:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:19:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:19:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:19:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:19:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:19:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:19:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:19:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:19:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:19:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:19:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:19:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:19:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:19:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:19:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:19:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:19:45.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:19:45.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:19:45.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:19:45.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_data_pool 2026-02-20T21:19:45.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-02-20T21:19:45.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:19:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:19:45.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:19:45.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_data_pool 2026-02-20T21:19:45.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:19:45.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:45.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:19:45.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:19:45.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:19:45.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:19:45.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:19:45.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:19:45.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:19:45.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:19:45.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:19:45.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:45.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:19:45.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:19:45.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:19:45.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:19:45.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:19:45.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:19:45.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:19:45.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:19:45.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:19:45.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:19:45.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:45.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:19:45.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:19:45.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:19:45.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:19:45.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:19:45.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:19:45.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:19:45.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:19:45.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:19:45.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:19:45.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_data_pool 2026-02-20T21:19:45.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-02-20T21:19:45.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:19:45.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:19:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:19:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_data_pool 2026-02-20T21:19:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:19:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:19:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:19:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:19:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:19:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:19:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:19:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:19:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:19:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:19:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:19:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:19:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:19:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:19:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:19:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:19:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:19:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:19:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:19:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:19:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:19:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:19:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:19:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:19:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:19:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:19:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:19:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:19:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:19:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:19:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_data_pool 2026-02-20T21:19:45.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-02-20T21:19:45.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:19:45.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:19:45.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:19:45.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 1 2026-02-20T21:19:46.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:19:46.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:19:46.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:19:46.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_data_pool 2026-02-20T21:19:46.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:19:46.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:46.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:19:46.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:19:46.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:19:46.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:19:46.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:19:46.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:19:46.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:19:46.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:19:46.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:19:46.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:46.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:19:46.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:19:46.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:19:46.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:19:46.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:19:46.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:19:46.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:19:46.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:19:46.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:19:46.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:19:46.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:46.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:19:46.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:19:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:19:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:19:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:19:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:19:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:19:46.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:19:46.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:19:46.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:19:46.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_data_pool 2026-02-20T21:19:46.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-02-20T21:19:46.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:19:46.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:19:46.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:19:46.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_data_pool 2026-02-20T21:19:46.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:19:46.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:19:46.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:19:46.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:19:46.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:19:46.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:19:46.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:19:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:19:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:19:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:19:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:19:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:19:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:19:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:19:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:19:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:19:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:19:46.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:19:46.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:19:46.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:19:46.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:19:46.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:19:46.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:19:46.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:19:46.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:19:46.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:19:46.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:19:46.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:19:46.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:19:46.500 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:19:46.500 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:19:46.500 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:19:46.500 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_data_pool 2026-02-20T21:19:46.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:19:46.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:19:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:19:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:19:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:19:46.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:19:46.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:19:46.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:46.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_data_pool cluster1 mirror test_data_pool 2026-02-20T21:19:46.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:19:46.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:19:46.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test_data_pool 2026-02-20T21:19:46.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 info mirror/test_data_pool 2026-02-20T21:19:46.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ awk '$1 == "data_pool:" {print $2}' 2026-02-20T21:19:46.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ data_pool=mirror_parent 2026-02-20T21:19:46.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test mirror_parent = mirror_parent 2026-02-20T21:19:46.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_snapshot cluster2 mirror test_data_pool snap1 2026-02-20T21:19:46.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:46.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:46.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:46.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap1 2026-02-20T21:19:46.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap create mirror/test_data_pool@snap1 2026-02-20T21:19:46.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-20T21:19:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test_data_pool 100 2026-02-20T21:19:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:19:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:19:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:19:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:19:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test_data_pool --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:19:46.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:19:47.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:19:47.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5000.08 bytes/sec: 20 MiB/s 2026-02-20T21:19:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_snapshot cluster2 mirror test_data_pool snap2 2026-02-20T21:19:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap2 2026-02-20T21:19:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap create mirror/test_data_pool@snap2 2026-02-20T21:19:47.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-20T21:19:47.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test_data_pool 100 2026-02-20T21:19:47.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:47.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:19:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:19:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:19:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:19:47.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test_data_pool --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:19:48.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:19:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:19:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5000.08 bytes/sec: 20 MiB/s 2026-02-20T21:19:48.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test_data_pool 2026-02-20T21:19:48.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:19:48.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:48.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:19:48.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:19:48.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:48.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:19:48.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:19:48.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test_data_pool 2026-02-20T21:19:48.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:48.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:48.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:48.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_data_pool 2026-02-20T21:19:48.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 15 2026-02-20T21:19:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test_data_pool 2026-02-20T21:19:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:19:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:19:48.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:19:48.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:48.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test_data_pool.status 2026-02-20T21:19:48.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test_data_pool.status 2026-02-20T21:19:48.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:19:48.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_data_pool.status 2026-02-20T21:19:48.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test_data_pool.status 2026-02-20T21:19:48.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test_data_pool.status 2026-02-20T21:19:48.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:19:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool.status 2026-02-20T21:19:48.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_data_pool /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_data_pool.status 2026-02-20T21:19:48.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:48.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:48.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:48.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_data_pool.status 2026-02-20T21:19:48.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_data_pool --format xml 2026-02-20T21:19:48.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:19:48.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:19:48.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=15 2026-02-20T21:19:48.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:19:48.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:19:48.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:19:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_data_pool /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool.status 2026-02-20T21:19:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool.status 2026-02-20T21:19:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_data_pool --format xml 2026-02-20T21:19:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:19:48.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:19:48.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=12 2026-02-20T21:19:48.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 15 = 12 2026-02-20T21:19:48.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:19:48.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.4 2026-02-20T21:19:48.821 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_data_pool /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool.status 2026-02-20T21:19:48.821 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:48.821 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:48.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:48.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool.status 2026-02-20T21:19:48.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_data_pool --format xml 2026-02-20T21:19:48.822 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:19:48.854 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:19:48.856 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=12 2026-02-20T21:19:48.856 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 15 = 12 2026-02-20T21:19:48.856 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:19:48.856 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.8 2026-02-20T21:19:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_data_pool /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool.status 2026-02-20T21:19:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool.status 2026-02-20T21:19:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_data_pool --format xml 2026-02-20T21:19:49.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:19:49.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:19:49.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:19:49.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:-:1.1: Document is empty 2026-02-20T21:19:49.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 2026-02-20T21:19:49.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:^ 2026-02-20T21:19:49.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id= 2026-02-20T21:19:49.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 15 = '' 2026-02-20T21:19:49.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:19:49.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1.6 2026-02-20T21:19:51.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_data_pool /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool.status 2026-02-20T21:19:51.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:51.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:51.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:51.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool.status 2026-02-20T21:19:51.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_data_pool --format xml 2026-02-20T21:19:51.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:19:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:19:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=15 2026-02-20T21:19:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 15 = 15 2026-02-20T21:19:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_data_pool 2026-02-20T21:19:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:19:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_data_pool up+replaying 2026-02-20T21:19:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:19:51.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:19:51.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:19:51.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:19:51.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:19:52.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_data_pool up+replaying '' '' 2026-02-20T21:19:52.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:52.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:52.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:52.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:19:52.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:19:52.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:19:52.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:19:52.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:19:52.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_data_pool 2026-02-20T21:19:52.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_data_pool: 2026-02-20T21:19:52.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 089510a4-b64d-4031-a8a6-893deb802b04 2026-02-20T21:19:52.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:19:52.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622384,"replay_state":"idle"} 2026-02-20T21:19:52.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:19:52.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:19:46 2026-02-20T21:19:52.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:19:52.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:19:52.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:19:52.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:19:52.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:19:51' 2026-02-20T21:19:52.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:19:52.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:19:52.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:19:52.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622384,"replay_state":"idle"} 2026-02-20T21:19:52.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:19:52.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:19:52.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:19:52.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:19:52.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:19:52.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:19:52.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:19:52.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:19:52.567 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:19:46' 2026-02-20T21:19:52.567 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_data_pool up+replaying '2026-02-20 21:19:46' 2026-02-20T21:19:52.567 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:19:52.567 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:52.568 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:52.568 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:19:52.568 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:19:46' 2026-02-20T21:19:52.568 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:19:52.568 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:19:52.568 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:19:52.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:19:28cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:19:51test_data_pool089510a4-b64d-4031-a8a6-893deb802b04up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622384,"replay_state":"idle"}44114415mirror.1trial1992026-02-20 21:19:46cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:19:51' 2026-02-20T21:19:52.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:19:52.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_data_pool'\'']/last_update' 2026-02-20T21:19:52.607 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:19:46' 2026-02-20T21:19:52.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_data_pool'\'']/state' 2026-02-20T21:19:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:19:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:19:52.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:19:52.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:19:52.611 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:52.611 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:52.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror mirror test_data_pool@snap1 2026-02-20T21:19:52.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:19:52.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:19:52.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:52.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:19:52.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:19:52.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool@snap1 2026-02-20T21:19:52.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test_data_pool@snap1.export 2026-02-20T21:19:52.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test_data_pool@snap1.export 2026-02-20T21:19:52.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:19:52.613 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_data_pool@snap1.export 2026-02-20T21:19:52.613 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test_data_pool@snap1.export 2026-02-20T21:19:52.613 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test_data_pool@snap1.export 2026-02-20T21:19:52.613 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:19:52.614 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool@snap1.export 2026-02-20T21:19:52.614 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_data_pool@snap1.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool@snap1.export 2026-02-20T21:19:52.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/test_data_pool@snap1 /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_data_pool@snap1.export 2026-02-20T21:19:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:19:52.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/test_data_pool@snap1 /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool@snap1.export 2026-02-20T21:19:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:19:52.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_data_pool@snap1.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool@snap1.export 2026-02-20T21:19:52.777 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_data_pool@snap1.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool@snap1.export 2026-02-20T21:19:52.795 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:52.795 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror mirror test_data_pool@snap2 2026-02-20T21:19:52.795 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:19:52.795 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:19:52.795 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:52.795 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:19:52.795 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:19:52.795 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool@snap2 2026-02-20T21:19:52.795 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test_data_pool@snap2.export 2026-02-20T21:19:52.796 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test_data_pool@snap2.export 2026-02-20T21:19:52.796 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:19:52.796 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_data_pool@snap2.export 2026-02-20T21:19:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test_data_pool@snap2.export 2026-02-20T21:19:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test_data_pool@snap2.export 2026-02-20T21:19:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:19:52.798 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool@snap2.export 2026-02-20T21:19:52.798 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_data_pool@snap2.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool@snap2.export 2026-02-20T21:19:52.798 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/test_data_pool@snap2 /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_data_pool@snap2.export 2026-02-20T21:19:52.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:19:52.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/test_data_pool@snap2 /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool@snap2.export 2026-02-20T21:19:52.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:19:52.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_data_pool@snap2.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool@snap2.export 2026-02-20T21:19:53.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_data_pool@snap2.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool@snap2.export 2026-02-20T21:19:53.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror mirror test_data_pool 2026-02-20T21:19:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:19:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:19:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:19:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:19:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test_data_pool.export 2026-02-20T21:19:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test_data_pool.export 2026-02-20T21:19:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:19:53.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_data_pool.export 2026-02-20T21:19:53.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test_data_pool.export 2026-02-20T21:19:53.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test_data_pool.export 2026-02-20T21:19:53.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:19:53.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool.export 2026-02-20T21:19:53.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_data_pool.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool.export 2026-02-20T21:19:53.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/test_data_pool /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_data_pool.export 2026-02-20T21:19:53.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:19:53.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/test_data_pool /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool.export 2026-02-20T21:19:53.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:19:53.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_data_pool.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool.export 2026-02-20T21:19:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_data_pool.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_data_pool.export 2026-02-20T21:19:53.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:53.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster2 mirror test_data_pool 2026-02-20T21:19:53.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:53.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:53.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:53.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:19:53.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:19:53.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster2 mirror test_data_pool 2026-02-20T21:19:53.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:53.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:53.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:19:53.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_data_pool 2026-02-20T21:19:55.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-02-20T21:19:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rm mirror/test_data_pool 2026-02-20T21:19:57.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:19:57.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:57.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: disable mirroring / delete non-primary image' 2026-02-20T21:19:57.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:19:57.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:19:57.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:19:57 TEST: disable mirroring / delete non-primary image 2026-02-20T21:19:57.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:19:57 TEST: disable mirroring / delete non-primary image 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image2=test2 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image3=test3 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image4=test4 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image5=test5 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image_and_enable_mirror cluster2 mirror test2 snapshot 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image cluster2 mirror test2 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=128 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:19:57.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,object-map,fast-diff mirror/test2 2026-02-20T21:19:57.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:19:57.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:57.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:19:57.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:19:57.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror test2 snapshot 2026-02-20T21:19:57.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:57.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:57.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:19:57.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:19:57.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test2 snapshot 2026-02-20T21:19:58.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:19:58.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/test2 2026-02-20T21:19:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test2': 2026-02-20T21:19:58.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:19:58.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:19:58.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:19:58.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 1637fb8cdea8 2026-02-20T21:19:58.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.1637fb8cdea8 2026-02-20T21:19:58.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:19:58.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:19:58.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:19:58.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:19:58.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:19:57 2026 2026-02-20T21:19:58.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:19:57 2026 2026-02-20T21:19:58.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:19:57 2026 2026-02-20T21:19:58.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:19:58.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:19:58.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 5c74a89f-0aa0-4cef-ac15-25e1d75d3add 2026-02-20T21:19:58.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:19:58.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test2 100 2026-02-20T21:19:58.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:58.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:58.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:19:58.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:19:58.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:19:58.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:19:58.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:19:58.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test2 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:19:58.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:19:58.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:19:58.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5882.44 bytes/sec: 23 MiB/s 2026-02-20T21:19:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_snapshot cluster2 mirror test2 snap1 2026-02-20T21:19:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:19:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap1 2026-02-20T21:19:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap create mirror/test2@snap1 2026-02-20T21:19:59.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-20T21:19:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_snapshot cluster2 mirror test2 snap2 2026-02-20T21:19:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:19:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:19:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:19:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap2 2026-02-20T21:19:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap create mirror/test2@snap2 2026-02-20T21:20:00.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-20T21:20:00.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' test2 = test4 ']' 2026-02-20T21:20:00.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' test2 = test5 ']' 2026-02-20T21:20:00.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test2 100 2026-02-20T21:20:00.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:00.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:00.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:00.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:20:00.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:20:00.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:00.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:20:00.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test2 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:20:00.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:20:00.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:20:00.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5263.24 bytes/sec: 21 MiB/s 2026-02-20T21:20:00.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:20:00.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test2 2026-02-20T21:20:00.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:00.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:00.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:00.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test2 2026-02-20T21:20:01.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 161 2026-02-20T21:20:01.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror test2 present 2026-02-20T21:20:01.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:01.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:01.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:01.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=present 2026-02-20T21:20:01.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:20:01.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:20:01.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:01.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test2 2026-02-20T21:20:01.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:20:01.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:20:01.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test2 2026-02-20T21:20:01.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-02-20T21:20:01.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:20:01.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image_id=113f256b47e4 2026-02-20T21:20:01.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:20:01.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:20:01.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test2 present 113f256b47e4 2026-02-20T21:20:01.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:01.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:01.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:01.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=present 2026-02-20T21:20:01.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id=113f256b47e4 2026-02-20T21:20:01.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:20:01.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:20:01.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test2 2026-02-20T21:20:01.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:20:01.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:20:01.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test2 2026-02-20T21:20:01.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-02-20T21:20:01.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:20:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id=113f256b47e4 2026-02-20T21:20:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n 113f256b47e4 2026-02-20T21:20:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -z 113f256b47e4 -o 113f256b47e4 = 113f256b47e4 2026-02-20T21:20:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=present 2026-02-20T21:20:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test present = present 2026-02-20T21:20:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snap_present cluster1 mirror test2 snap2 2026-02-20T21:20:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap_name=snap2 2026-02-20T21:20:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:20:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:20:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:20:02.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 info mirror/test2@snap2 2026-02-20T21:20:02.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:error setting snapshot context: (2) No such file or directory 2026-02-20T21:20:02.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ continue 2026-02-20T21:20:02.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:20:02.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:20:04.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 info mirror/test2@snap2 2026-02-20T21:20:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test2': 2026-02-20T21:20:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:20:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:20:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 4 2026-02-20T21:20:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 113f256b47e4 2026-02-20T21:20:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.113f256b47e4 2026-02-20T21:20:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:20:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff, operations, non-primary 2026-02-20T21:20:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: snap-trash 2026-02-20T21:20:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:20:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:20:00 2026 2026-02-20T21:20:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:20:00 2026 2026-02-20T21:20:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:20:00 2026 2026-02-20T21:20:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: protected: False 2026-02-20T21:20:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:20:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:20:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 5c74a89f-0aa0-4cef-ac15-25e1d75d3add 2026-02-20T21:20:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: false 2026-02-20T21:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-20T21:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image_and_enable_mirror cluster2 mirror test3 snapshot 2026-02-20T21:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test3 2026-02-20T21:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image cluster2 mirror test3 2026-02-20T21:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test3 2026-02-20T21:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=128 2026-02-20T21:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:20:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,object-map,fast-diff mirror/test3 2026-02-20T21:20:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:20:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:20:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror test3 snapshot 2026-02-20T21:20:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:04.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:04.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test3 2026-02-20T21:20:04.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:20:04.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test3 snapshot 2026-02-20T21:20:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:20:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/test3 2026-02-20T21:20:05.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test3': 2026-02-20T21:20:05.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:20:05.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:20:05.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:20:05.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 164fc1a5d088 2026-02-20T21:20:05.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.164fc1a5d088 2026-02-20T21:20:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:20:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:20:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:20:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:20:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:20:04 2026 2026-02-20T21:20:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:20:04 2026 2026-02-20T21:20:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:20:04 2026 2026-02-20T21:20:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:20:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:20:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 676342f8-18c4-4a8b-a54f-81f2da76cd0d 2026-02-20T21:20:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:20:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test3 100 2026-02-20T21:20:05.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:05.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:05.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test3 2026-02-20T21:20:05.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:20:05.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:20:05.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:05.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:20:05.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test3 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:20:05.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:20:05.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:20:05.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5882.44 bytes/sec: 23 MiB/s 2026-02-20T21:20:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_snapshot cluster2 mirror test3 snap1 2026-02-20T21:20:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test3 2026-02-20T21:20:05.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap1 2026-02-20T21:20:05.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap create mirror/test3@snap1 2026-02-20T21:20:06.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-20T21:20:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_snapshot cluster2 mirror test3 snap2 2026-02-20T21:20:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test3 2026-02-20T21:20:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap2 2026-02-20T21:20:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap create mirror/test3@snap2 2026-02-20T21:20:07.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-20T21:20:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' test3 = test4 ']' 2026-02-20T21:20:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' test3 = test5 ']' 2026-02-20T21:20:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test3 100 2026-02-20T21:20:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test3 2026-02-20T21:20:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:20:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:20:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:20:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test3 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:20:07.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:20:07.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:20:07.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5000.08 bytes/sec: 20 MiB/s 2026-02-20T21:20:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:20:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test3 2026-02-20T21:20:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test3 2026-02-20T21:20:07.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test3 2026-02-20T21:20:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 165 2026-02-20T21:20:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror test3 present 2026-02-20T21:20:08.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:08.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:08.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test3 2026-02-20T21:20:08.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=present 2026-02-20T21:20:08.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:20:08.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:20:08.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:08.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test3 2026-02-20T21:20:08.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:20:08.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:20:08.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test3 2026-02-20T21:20:08.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test3 2026-02-20T21:20:08.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:20:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image_id=1149448698a6 2026-02-20T21:20:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:20:08.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:20:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test3 present 1149448698a6 2026-02-20T21:20:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test3 2026-02-20T21:20:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=present 2026-02-20T21:20:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id=1149448698a6 2026-02-20T21:20:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:20:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:20:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test3 2026-02-20T21:20:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:20:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:20:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test3 2026-02-20T21:20:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test3 2026-02-20T21:20:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:20:08.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id=1149448698a6 2026-02-20T21:20:08.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n 1149448698a6 2026-02-20T21:20:08.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -z 1149448698a6 -o 1149448698a6 = 1149448698a6 2026-02-20T21:20:08.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=present 2026-02-20T21:20:08.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test present = present 2026-02-20T21:20:08.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:08.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snap_present cluster1 mirror test3 snap2 2026-02-20T21:20:08.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:08.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:08.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test3 2026-02-20T21:20:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap_name=snap2 2026-02-20T21:20:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:20:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:20:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:20:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 info mirror/test3@snap2 2026-02-20T21:20:09.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:error setting snapshot context: (2) No such file or directory 2026-02-20T21:20:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ continue 2026-02-20T21:20:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:20:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:20:11.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 info mirror/test3@snap2 2026-02-20T21:20:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test3': 2026-02-20T21:20:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:20:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:20:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 4 2026-02-20T21:20:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 1149448698a6 2026-02-20T21:20:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.1149448698a6 2026-02-20T21:20:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:20:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff, operations, non-primary 2026-02-20T21:20:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: snap-trash 2026-02-20T21:20:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:20:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:20:07 2026 2026-02-20T21:20:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:20:07 2026 2026-02-20T21:20:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:20:07 2026 2026-02-20T21:20:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: protected: False 2026-02-20T21:20:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:20:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:20:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 676342f8-18c4-4a8b-a54f-81f2da76cd0d 2026-02-20T21:20:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: false 2026-02-20T21:20:11.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:11.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-20T21:20:11.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image_and_enable_mirror cluster2 mirror test4 snapshot 2026-02-20T21:20:11.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test4 2026-02-20T21:20:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:20:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:20:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image cluster2 mirror test4 2026-02-20T21:20:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:11.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:11.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test4 2026-02-20T21:20:11.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:11.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=128 2026-02-20T21:20:11.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:20:11.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,object-map,fast-diff mirror/test4 2026-02-20T21:20:11.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:20:11.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:11.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:20:11.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:11.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror test4 snapshot 2026-02-20T21:20:11.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test4 2026-02-20T21:20:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:20:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test4 snapshot 2026-02-20T21:20:11.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:20:11.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/test4 2026-02-20T21:20:11.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test4': 2026-02-20T21:20:11.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:20:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:20:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:20:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 16672f98bc58 2026-02-20T21:20:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.16672f98bc58 2026-02-20T21:20:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:20:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:20:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:20:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:20:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:20:11 2026 2026-02-20T21:20:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:20:11 2026 2026-02-20T21:20:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:20:11 2026 2026-02-20T21:20:11.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:20:11.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:20:11.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 9751567e-049e-4f8d-be5b-924bc0c3e1f8 2026-02-20T21:20:11.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:20:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test4 100 2026-02-20T21:20:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test4 2026-02-20T21:20:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:20:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:20:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:20:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test4 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:20:12.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:20:12.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:20:12.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5000.08 bytes/sec: 20 MiB/s 2026-02-20T21:20:12.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_snapshot cluster2 mirror test4 snap1 2026-02-20T21:20:12.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:12.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:12.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test4 2026-02-20T21:20:12.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap1 2026-02-20T21:20:12.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap create mirror/test4@snap1 2026-02-20T21:20:12.950 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-20T21:20:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_snapshot cluster2 mirror test4 snap2 2026-02-20T21:20:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test4 2026-02-20T21:20:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap2 2026-02-20T21:20:12.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap create mirror/test4@snap2 2026-02-20T21:20:13.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-20T21:20:13.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' test4 = test4 ']' 2026-02-20T21:20:13.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ protect_snapshot cluster2 mirror test4 snap1 2026-02-20T21:20:13.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:13.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:13.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test4 2026-02-20T21:20:13.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap1 2026-02-20T21:20:13.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap protect mirror/test4@snap1 2026-02-20T21:20:13.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ protect_snapshot cluster2 mirror test4 snap2 2026-02-20T21:20:13.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:13.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:13.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test4 2026-02-20T21:20:13.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap2 2026-02-20T21:20:13.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap protect mirror/test4@snap2 2026-02-20T21:20:14.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test4 100 2026-02-20T21:20:14.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:14.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:14.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test4 2026-02-20T21:20:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:20:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:20:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:20:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test4 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:20:14.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:20:14.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:20:14.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5000.08 bytes/sec: 20 MiB/s 2026-02-20T21:20:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:20:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test4 2026-02-20T21:20:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test4 2026-02-20T21:20:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test4 2026-02-20T21:20:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 169 2026-02-20T21:20:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror test4 present 2026-02-20T21:20:14.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test4 2026-02-20T21:20:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=present 2026-02-20T21:20:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:20:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:20:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test4 2026-02-20T21:20:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:20:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:20:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test4 2026-02-20T21:20:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-02-20T21:20:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:20:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image_id=1151db3a370f 2026-02-20T21:20:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:20:14.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:20:15.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test4 present 1151db3a370f 2026-02-20T21:20:15.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:15.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:15.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test4 2026-02-20T21:20:15.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=present 2026-02-20T21:20:15.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id=1151db3a370f 2026-02-20T21:20:15.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:20:15.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:20:15.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test4 2026-02-20T21:20:15.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:20:15.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:20:15.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test4 2026-02-20T21:20:15.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-02-20T21:20:15.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:20:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id=1151db3a370f 2026-02-20T21:20:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n 1151db3a370f 2026-02-20T21:20:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -z 1151db3a370f -o 1151db3a370f = 1151db3a370f 2026-02-20T21:20:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=present 2026-02-20T21:20:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test present = present 2026-02-20T21:20:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snap_present cluster1 mirror test4 snap2 2026-02-20T21:20:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test4 2026-02-20T21:20:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap_name=snap2 2026-02-20T21:20:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:20:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:20:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:20:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 info mirror/test4@snap2 2026-02-20T21:20:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test4': 2026-02-20T21:20:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:20:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:20:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 3 2026-02-20T21:20:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 1151db3a370f 2026-02-20T21:20:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.1151db3a370f 2026-02-20T21:20:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:20:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:20:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:20:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:20:16.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:20:13 2026 2026-02-20T21:20:16.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:20:13 2026 2026-02-20T21:20:16.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:20:13 2026 2026-02-20T21:20:16.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: protected: True 2026-02-20T21:20:16.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:20:16.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:20:16.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 9751567e-049e-4f8d-be5b-924bc0c3e1f8 2026-02-20T21:20:16.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: false 2026-02-20T21:20:16.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:16.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-20T21:20:16.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image_and_enable_mirror cluster2 mirror test5 snapshot 2026-02-20T21:20:16.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:16.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:16.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:16.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:16.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test5 2026-02-20T21:20:16.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:20:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:20:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image cluster2 mirror test5 2026-02-20T21:20:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test5 2026-02-20T21:20:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=128 2026-02-20T21:20:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:20:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,object-map,fast-diff mirror/test5 2026-02-20T21:20:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:20:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:20:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror test5 snapshot 2026-02-20T21:20:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test5 2026-02-20T21:20:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:20:16.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test5 snapshot 2026-02-20T21:20:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:20:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/test5 2026-02-20T21:20:16.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test5': 2026-02-20T21:20:16.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:20:16.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:20:16.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:20:16.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 16854c7952c6 2026-02-20T21:20:16.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.16854c7952c6 2026-02-20T21:20:16.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:20:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:20:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:20:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:20:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:20:16 2026 2026-02-20T21:20:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:20:16 2026 2026-02-20T21:20:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:20:16 2026 2026-02-20T21:20:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:20:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:20:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 600a88ca-a374-4c12-a33d-3ac6368f89dd 2026-02-20T21:20:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:20:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test5 100 2026-02-20T21:20:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:16.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:16.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test5 2026-02-20T21:20:16.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:20:16.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:20:16.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:16.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:20:16.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test5 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:20:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:20:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:20:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5000.08 bytes/sec: 20 MiB/s 2026-02-20T21:20:17.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_snapshot cluster2 mirror test5 snap1 2026-02-20T21:20:17.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:17.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:17.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test5 2026-02-20T21:20:17.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap1 2026-02-20T21:20:17.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap create mirror/test5@snap1 2026-02-20T21:20:17.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-20T21:20:17.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_snapshot cluster2 mirror test5 snap2 2026-02-20T21:20:17.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:17.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:17.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test5 2026-02-20T21:20:17.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap2 2026-02-20T21:20:17.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap create mirror/test5@snap2 2026-02-20T21:20:18.960 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-20T21:20:18.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' test5 = test4 ']' 2026-02-20T21:20:18.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' test5 = test5 ']' 2026-02-20T21:20:18.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ protect_snapshot cluster2 mirror test5 snap1 2026-02-20T21:20:18.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:18.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:18.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test5 2026-02-20T21:20:18.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap1 2026-02-20T21:20:18.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap protect mirror/test5@snap1 2026-02-20T21:20:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ protect_snapshot cluster2 mirror test5 snap2 2026-02-20T21:20:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test5 2026-02-20T21:20:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap2 2026-02-20T21:20:18.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap protect mirror/test5@snap2 2026-02-20T21:20:19.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test5 100 2026-02-20T21:20:19.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:19.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:19.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test5 2026-02-20T21:20:19.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:20:19.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:20:19.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:19.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:20:19.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test5 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:20:19.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:20:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:20:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5882.44 bytes/sec: 23 MiB/s 2026-02-20T21:20:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:20:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test5 2026-02-20T21:20:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test5 2026-02-20T21:20:19.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test5 2026-02-20T21:20:19.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 173 2026-02-20T21:20:19.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror test5 present 2026-02-20T21:20:19.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:19.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test5 2026-02-20T21:20:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=present 2026-02-20T21:20:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:20:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:20:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test5 2026-02-20T21:20:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:20:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:20:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test5 2026-02-20T21:20:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test5 2026-02-20T21:20:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:20:20.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image_id=1139126333cc 2026-02-20T21:20:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:20:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:20:20.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test5 present 1139126333cc 2026-02-20T21:20:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test5 2026-02-20T21:20:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=present 2026-02-20T21:20:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id=1139126333cc 2026-02-20T21:20:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:20:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:20:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test5 2026-02-20T21:20:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:20:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:20:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test5 2026-02-20T21:20:20.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test5 2026-02-20T21:20:20.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:20:20.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id=1139126333cc 2026-02-20T21:20:20.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n 1139126333cc 2026-02-20T21:20:20.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -z 1139126333cc -o 1139126333cc = 1139126333cc 2026-02-20T21:20:20.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=present 2026-02-20T21:20:20.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test present = present 2026-02-20T21:20:20.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:20.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snap_present cluster1 mirror test5 snap2 2026-02-20T21:20:20.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:20.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:20.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test5 2026-02-20T21:20:20.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap_name=snap2 2026-02-20T21:20:20.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:20:20.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:20:20.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:20:21.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 info mirror/test5@snap2 2026-02-20T21:20:21.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test5': 2026-02-20T21:20:21.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:20:21.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:20:21.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 3 2026-02-20T21:20:21.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 1139126333cc 2026-02-20T21:20:21.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.1139126333cc 2026-02-20T21:20:21.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:20:21.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:20:21.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:20:21.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:20:21.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:20:18 2026 2026-02-20T21:20:21.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:20:18 2026 2026-02-20T21:20:21.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:20:18 2026 2026-02-20T21:20:21.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: protected: True 2026-02-20T21:20:21.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:20:21.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:20:21.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 600a88ca-a374-4c12-a33d-3ac6368f89dd 2026-02-20T21:20:21.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: false 2026-02-20T21:20:21.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:21.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_pool_mirror_mode cluster2 mirror image 2026-02-20T21:20:21.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:21.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:21.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=image 2026-02-20T21:20:21.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror pool enable mirror image 2026-02-20T21:20:21.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd: mirroring is already configured for image mode 2026-02-20T21:20:21.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in ${image2} ${image4} 2026-02-20T21:20:21.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ disable_mirror cluster2 mirror test2 2026-02-20T21:20:21.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:21.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:21.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:21.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/test2 2026-02-20T21:20:22.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring disabled 2026-02-20T21:20:22.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in ${image2} ${image4} 2026-02-20T21:20:22.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ disable_mirror cluster2 mirror test4 2026-02-20T21:20:22.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:22.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:22.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test4 2026-02-20T21:20:22.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/test4 2026-02-20T21:20:24.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring disabled 2026-02-20T21:20:24.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ unprotect_snapshot cluster2 mirror test5 snap1 2026-02-20T21:20:24.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:24.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:24.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test5 2026-02-20T21:20:24.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap1 2026-02-20T21:20:24.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap unprotect mirror/test5@snap1 2026-02-20T21:20:25.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ unprotect_snapshot cluster2 mirror test5 snap2 2026-02-20T21:20:25.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:25.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:25.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test5 2026-02-20T21:20:25.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap2 2026-02-20T21:20:25.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap unprotect mirror/test5@snap2 2026-02-20T21:20:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in ${image3} ${image5} 2026-02-20T21:20:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_snapshot cluster2 mirror test3 snap1 2026-02-20T21:20:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test3 2026-02-20T21:20:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap1 2026-02-20T21:20:25.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rm mirror/test3@snap1 2026-02-20T21:20:25.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing snap: 100% complete...done. 2026-02-20T21:20:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_snapshot cluster2 mirror test3 snap2 2026-02-20T21:20:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test3 2026-02-20T21:20:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap2 2026-02-20T21:20:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rm mirror/test3@snap2 2026-02-20T21:20:26.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing snap: 100% complete...done. 2026-02-20T21:20:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster2 mirror test3 2026-02-20T21:20:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test3 2026-02-20T21:20:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:20:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:20:26.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster2 mirror test3 2026-02-20T21:20:26.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:26.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:26.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test3 2026-02-20T21:20:26.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 snap purge mirror/test3 2026-02-20T21:20:27.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rm mirror/test3 2026-02-20T21:20:28.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:20:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in ${image3} ${image5} 2026-02-20T21:20:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_snapshot cluster2 mirror test5 snap1 2026-02-20T21:20:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test5 2026-02-20T21:20:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap1 2026-02-20T21:20:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rm mirror/test5@snap1 2026-02-20T21:20:29.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing snap: 100% complete...done. 2026-02-20T21:20:29.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_snapshot cluster2 mirror test5 snap2 2026-02-20T21:20:29.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:29.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:29.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test5 2026-02-20T21:20:29.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap2 2026-02-20T21:20:29.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rm mirror/test5@snap2 2026-02-20T21:20:30.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing snap: 100% complete...done. 2026-02-20T21:20:30.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster2 mirror test5 2026-02-20T21:20:30.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:30.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:30.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test5 2026-02-20T21:20:30.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:20:30.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:20:30.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster2 mirror test5 2026-02-20T21:20:30.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:30.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:30.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test5 2026-02-20T21:20:30.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 snap purge mirror/test5 2026-02-20T21:20:30.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rm mirror/test5 2026-02-20T21:20:32.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:20:32.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:32.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-20T21:20:32.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror test2 deleted 2026-02-20T21:20:32.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:32.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:32.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:32.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=deleted 2026-02-20T21:20:32.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:20:32.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:20:32.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:32.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test2 2026-02-20T21:20:32.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:20:32.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:20:32.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test2 2026-02-20T21:20:32.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-02-20T21:20:32.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:20:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test2: (2) No such file or directory 2026-02-20T21:20:32.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image_id= 2026-02-20T21:20:32.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:20:32.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:20:32.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test2 deleted '' 2026-02-20T21:20:32.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:32.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:32.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:32.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=deleted 2026-02-20T21:20:32.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:20:32.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:20:32.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:20:32.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test2 2026-02-20T21:20:32.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:20:32.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:20:32.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test2 2026-02-20T21:20:32.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-02-20T21:20:32.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:20:32.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test2: (2) No such file or directory 2026-02-20T21:20:32.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id= 2026-02-20T21:20:32.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:32.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test deleted = deleted 2026-02-20T21:20:32.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:32.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-20T21:20:32.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror test3 deleted 2026-02-20T21:20:32.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:32.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:32.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test3 2026-02-20T21:20:32.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=deleted 2026-02-20T21:20:32.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:20:32.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:20:32.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:32.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test3 2026-02-20T21:20:32.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:20:32.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:20:32.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test3 2026-02-20T21:20:32.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test3 2026-02-20T21:20:32.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:20:32.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test3: (2) No such file or directory 2026-02-20T21:20:32.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image_id= 2026-02-20T21:20:32.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:20:32.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:20:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test3 deleted '' 2026-02-20T21:20:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test3 2026-02-20T21:20:32.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=deleted 2026-02-20T21:20:32.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:20:32.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:20:32.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:20:32.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test3 2026-02-20T21:20:32.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:20:32.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:20:32.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test3 2026-02-20T21:20:32.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test3 2026-02-20T21:20:32.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:20:32.686 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test3: (2) No such file or directory 2026-02-20T21:20:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id= 2026-02-20T21:20:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test deleted = deleted 2026-02-20T21:20:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-20T21:20:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror test4 deleted 2026-02-20T21:20:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test4 2026-02-20T21:20:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=deleted 2026-02-20T21:20:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:20:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:20:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test4 2026-02-20T21:20:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:20:32.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:20:32.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test4 2026-02-20T21:20:32.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-02-20T21:20:32.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:20:32.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test4: (2) No such file or directory 2026-02-20T21:20:32.713 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image_id= 2026-02-20T21:20:32.713 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:20:32.713 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:20:32.814 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test4 deleted '' 2026-02-20T21:20:32.815 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:32.815 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:32.815 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test4 2026-02-20T21:20:32.815 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=deleted 2026-02-20T21:20:32.815 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:20:32.815 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:20:32.815 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:20:32.815 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test4 2026-02-20T21:20:32.815 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:20:32.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:20:32.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test4 2026-02-20T21:20:32.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-02-20T21:20:32.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:20:32.836 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test4: (2) No such file or directory 2026-02-20T21:20:32.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id= 2026-02-20T21:20:32.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:32.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test deleted = deleted 2026-02-20T21:20:32.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:32.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-20T21:20:32.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror test5 deleted 2026-02-20T21:20:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test5 2026-02-20T21:20:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=deleted 2026-02-20T21:20:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:20:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:20:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test5 2026-02-20T21:20:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:20:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:20:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test5 2026-02-20T21:20:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test5 2026-02-20T21:20:32.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:20:32.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test5: (2) No such file or directory 2026-02-20T21:20:32.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image_id= 2026-02-20T21:20:32.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:20:32.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:20:32.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test5 deleted '' 2026-02-20T21:20:32.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:32.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:32.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test5 2026-02-20T21:20:32.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=deleted 2026-02-20T21:20:32.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:20:32.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:20:32.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:20:32.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test5 2026-02-20T21:20:32.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:20:32.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:20:32.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test5 2026-02-20T21:20:32.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test5 2026-02-20T21:20:32.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:20:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test5: (2) No such file or directory 2026-02-20T21:20:32.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id= 2026-02-20T21:20:32.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:32.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test deleted = deleted 2026-02-20T21:20:32.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:32.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:20:32.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: snapshot rename' 2026-02-20T21:20:32.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:20:32.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:20:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:20:32 TEST: snapshot rename 2026-02-20T21:20:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:20:32 TEST: snapshot rename 2026-02-20T21:20:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ snap_name=snap_rename 2026-02-20T21:20:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:20:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror test2 2026-02-20T21:20:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:20:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test2 snapshot 2026-02-20T21:20:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:20:33.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/test2 2026-02-20T21:20:33.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test2': 2026-02-20T21:20:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:20:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:20:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 3 2026-02-20T21:20:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 1637fb8cdea8 2026-02-20T21:20:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.1637fb8cdea8 2026-02-20T21:20:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:20:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:20:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:20:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:20:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:19:57 2026 2026-02-20T21:20:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:19:57 2026 2026-02-20T21:20:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:19:57 2026 2026-02-20T21:20:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:20:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:20:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: c6f9aa43-03a3-4c22-b0b7-00659c111fd4 2026-02-20T21:20:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:20:33.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_snapshot cluster2 mirror test2 snap_rename_0 2026-02-20T21:20:33.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:33.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:33.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:33.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_rename_0 2026-02-20T21:20:33.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap create mirror/test2@snap_rename_0 2026-02-20T21:20:34.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-20T21:20:34.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 1 20 2026-02-20T21:20:34.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:20:34.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ expr 1 - 1 2026-02-20T21:20:34.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_0 snap_rename_1 2026-02-20T21:20:34.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:34.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:34.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:34.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_rename_0 2026-02-20T21:20:34.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local new_snap=snap_rename_1 2026-02-20T21:20:34.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_0 mirror/test2@snap_rename_1 2026-02-20T21:20:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:20:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ expr 2 - 1 2026-02-20T21:20:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_1 snap_rename_2 2026-02-20T21:20:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_rename_1 2026-02-20T21:20:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local new_snap=snap_rename_2 2026-02-20T21:20:34.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_1 mirror/test2@snap_rename_2 2026-02-20T21:20:34.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:20:34.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ expr 3 - 1 2026-02-20T21:20:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_2 snap_rename_3 2026-02-20T21:20:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_rename_2 2026-02-20T21:20:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local new_snap=snap_rename_3 2026-02-20T21:20:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_2 mirror/test2@snap_rename_3 2026-02-20T21:20:34.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:20:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ expr 4 - 1 2026-02-20T21:20:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_3 snap_rename_4 2026-02-20T21:20:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_rename_3 2026-02-20T21:20:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local new_snap=snap_rename_4 2026-02-20T21:20:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_3 mirror/test2@snap_rename_4 2026-02-20T21:20:34.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:20:34.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ expr 5 - 1 2026-02-20T21:20:34.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_4 snap_rename_5 2026-02-20T21:20:34.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:34.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:34.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:34.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_rename_4 2026-02-20T21:20:34.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local new_snap=snap_rename_5 2026-02-20T21:20:34.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_4 mirror/test2@snap_rename_5 2026-02-20T21:20:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:20:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ expr 6 - 1 2026-02-20T21:20:34.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_5 snap_rename_6 2026-02-20T21:20:34.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:34.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:34.514 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:34.514 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_rename_5 2026-02-20T21:20:34.514 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local new_snap=snap_rename_6 2026-02-20T21:20:34.514 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_5 mirror/test2@snap_rename_6 2026-02-20T21:20:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:20:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ expr 7 - 1 2026-02-20T21:20:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_6 snap_rename_7 2026-02-20T21:20:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_rename_6 2026-02-20T21:20:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local new_snap=snap_rename_7 2026-02-20T21:20:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_6 mirror/test2@snap_rename_7 2026-02-20T21:20:34.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:20:34.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ expr 8 - 1 2026-02-20T21:20:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_7 snap_rename_8 2026-02-20T21:20:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_rename_7 2026-02-20T21:20:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local new_snap=snap_rename_8 2026-02-20T21:20:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_7 mirror/test2@snap_rename_8 2026-02-20T21:20:34.602 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:20:34.603 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ expr 9 - 1 2026-02-20T21:20:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_8 snap_rename_9 2026-02-20T21:20:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:34.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_rename_8 2026-02-20T21:20:34.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local new_snap=snap_rename_9 2026-02-20T21:20:34.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_8 mirror/test2@snap_rename_9 2026-02-20T21:20:34.631 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:20:34.632 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ expr 10 - 1 2026-02-20T21:20:34.633 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_9 snap_rename_10 2026-02-20T21:20:34.633 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:34.633 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:34.633 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:34.633 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_rename_9 2026-02-20T21:20:34.633 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local new_snap=snap_rename_10 2026-02-20T21:20:34.633 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_9 mirror/test2@snap_rename_10 2026-02-20T21:20:34.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:20:34.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ expr 11 - 1 2026-02-20T21:20:34.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_10 snap_rename_11 2026-02-20T21:20:34.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:34.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:34.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:34.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_rename_10 2026-02-20T21:20:34.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local new_snap=snap_rename_11 2026-02-20T21:20:34.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_10 mirror/test2@snap_rename_11 2026-02-20T21:20:34.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:20:34.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ expr 12 - 1 2026-02-20T21:20:34.692 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_11 snap_rename_12 2026-02-20T21:20:34.692 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:34.692 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:34.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:34.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_rename_11 2026-02-20T21:20:34.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local new_snap=snap_rename_12 2026-02-20T21:20:34.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_11 mirror/test2@snap_rename_12 2026-02-20T21:20:34.720 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:20:34.721 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ expr 13 - 1 2026-02-20T21:20:34.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_12 snap_rename_13 2026-02-20T21:20:34.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:34.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:34.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:34.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_rename_12 2026-02-20T21:20:34.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local new_snap=snap_rename_13 2026-02-20T21:20:34.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_12 mirror/test2@snap_rename_13 2026-02-20T21:20:34.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:20:34.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ expr 14 - 1 2026-02-20T21:20:34.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_13 snap_rename_14 2026-02-20T21:20:34.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:34.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:34.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:34.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_rename_13 2026-02-20T21:20:34.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local new_snap=snap_rename_14 2026-02-20T21:20:34.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_13 mirror/test2@snap_rename_14 2026-02-20T21:20:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:20:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ expr 15 - 1 2026-02-20T21:20:34.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_14 snap_rename_15 2026-02-20T21:20:34.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:34.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:34.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:34.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_rename_14 2026-02-20T21:20:34.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local new_snap=snap_rename_15 2026-02-20T21:20:34.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_14 mirror/test2@snap_rename_15 2026-02-20T21:20:34.809 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:20:34.810 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ expr 16 - 1 2026-02-20T21:20:34.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_15 snap_rename_16 2026-02-20T21:20:34.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:34.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:34.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:34.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_rename_15 2026-02-20T21:20:34.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local new_snap=snap_rename_16 2026-02-20T21:20:34.811 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_15 mirror/test2@snap_rename_16 2026-02-20T21:20:34.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:20:34.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ expr 17 - 1 2026-02-20T21:20:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_16 snap_rename_17 2026-02-20T21:20:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:34.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_rename_16 2026-02-20T21:20:34.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local new_snap=snap_rename_17 2026-02-20T21:20:34.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_16 mirror/test2@snap_rename_17 2026-02-20T21:20:34.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:20:34.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ expr 18 - 1 2026-02-20T21:20:34.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_17 snap_rename_18 2026-02-20T21:20:34.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:34.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:34.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:34.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_rename_17 2026-02-20T21:20:34.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local new_snap=snap_rename_18 2026-02-20T21:20:34.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_17 mirror/test2@snap_rename_18 2026-02-20T21:20:34.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:20:34.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ expr 19 - 1 2026-02-20T21:20:34.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_18 snap_rename_19 2026-02-20T21:20:34.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:34.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:34.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:34.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_rename_18 2026-02-20T21:20:34.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local new_snap=snap_rename_19 2026-02-20T21:20:34.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_18 mirror/test2@snap_rename_19 2026-02-20T21:20:34.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in `seq 1 20` 2026-02-20T21:20:34.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ expr 20 - 1 2026-02-20T21:20:34.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_19 snap_rename_20 2026-02-20T21:20:34.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:34.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:34.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:34.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap_rename_19 2026-02-20T21:20:34.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local new_snap=snap_rename_20 2026-02-20T21:20:34.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_19 mirror/test2@snap_rename_20 2026-02-20T21:20:34.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:20:34.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test2 2026-02-20T21:20:34.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:34.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:34.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:34.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test2 2026-02-20T21:20:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 176 2026-02-20T21:20:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snap_present cluster1 mirror test2 snap_rename_20 2026-02-20T21:20:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap_name=snap_rename_20 2026-02-20T21:20:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:20:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:20:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:20:36.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 info mirror/test2@snap_rename_20 2026-02-20T21:20:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:error setting snapshot context: (2) No such file or directory 2026-02-20T21:20:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ continue 2026-02-20T21:20:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:20:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:20:38.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 info mirror/test2@snap_rename_20 2026-02-20T21:20:38.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:error setting snapshot context: (2) No such file or directory 2026-02-20T21:20:38.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ continue 2026-02-20T21:20:38.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:20:38.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:20:42.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 info mirror/test2@snap_rename_20 2026-02-20T21:20:42.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test2': 2026-02-20T21:20:42.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:20:42.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:20:42.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 4 2026-02-20T21:20:42.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 113f91b4bfd7 2026-02-20T21:20:42.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.113f91b4bfd7 2026-02-20T21:20:42.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:20:42.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:20:42.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:20:42.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:20:42.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:20:35 2026 2026-02-20T21:20:42.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:20:35 2026 2026-02-20T21:20:42.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:20:35 2026 2026-02-20T21:20:42.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: protected: False 2026-02-20T21:20:42.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:20:42.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:20:42.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: c6f9aa43-03a3-4c22-b0b7-00659c111fd4 2026-02-20T21:20:42.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: false 2026-02-20T21:20:42.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:42.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ unprotect_snapshot cluster2 mirror test4 snap1 2026-02-20T21:20:42.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:42.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:42.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test4 2026-02-20T21:20:42.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap1 2026-02-20T21:20:42.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap unprotect mirror/test4@snap1 2026-02-20T21:20:42.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ unprotect_snapshot cluster2 mirror test4 snap2 2026-02-20T21:20:42.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:42.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:42.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test4 2026-02-20T21:20:42.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap2 2026-02-20T21:20:42.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap unprotect mirror/test4@snap2 2026-02-20T21:20:42.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in ${image2} ${image4} 2026-02-20T21:20:42.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster2 mirror test2 2026-02-20T21:20:42.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:42.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:42.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:42.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:20:42.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:20:42.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster2 mirror test2 2026-02-20T21:20:42.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:42.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:42.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test2 2026-02-20T21:20:42.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 snap purge mirror/test2 2026-02-20T21:20:44.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing all snapshots: 33% complete... Removing all snapshots: 66% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-02-20T21:20:44.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rm mirror/test2 2026-02-20T21:20:46.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:20:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for i in ${image2} ${image4} 2026-02-20T21:20:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster2 mirror test4 2026-02-20T21:20:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test4 2026-02-20T21:20:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:20:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:20:46.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster2 mirror test4 2026-02-20T21:20:46.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:46.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:46.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test4 2026-02-20T21:20:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 snap purge mirror/test4 2026-02-20T21:20:48.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-02-20T21:20:48.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rm mirror/test4 2026-02-20T21:20:48.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:20:48.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:48.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -z '' ']' 2026-02-20T21:20:48.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: demote image while daemon is offline' 2026-02-20T21:20:48.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:20:48.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:20:48.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:20:48 TEST: demote image while daemon is offline 2026-02-20T21:20:48.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:20:48 TEST: demote image while daemon is offline 2026-02-20T21:20:48.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image=test_demote_image 2026-02-20T21:20:48.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image_and_enable_mirror cluster2 mirror test_demote_image snapshot 2026-02-20T21:20:48.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:48.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:48.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:48.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:48.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:20:48.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:48.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:20:48.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:20:48.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:48.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image cluster2 mirror test_demote_image 2026-02-20T21:20:48.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:48.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:48.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:48.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:48.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:20:48.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:20:48.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=128 2026-02-20T21:20:48.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:20:48.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,object-map,fast-diff mirror/test_demote_image 2026-02-20T21:20:48.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:20:48.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:48.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:20:48.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:48.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror test_demote_image snapshot 2026-02-20T21:20:48.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:48.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:48.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:20:48.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:20:48.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_demote_image snapshot 2026-02-20T21:20:49.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:20:49.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/test_demote_image 2026-02-20T21:20:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test_demote_image': 2026-02-20T21:20:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:20:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:20:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:20:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 1723b2459e39 2026-02-20T21:20:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.1723b2459e39 2026-02-20T21:20:49.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:20:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:20:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:20:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:20:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:20:48 2026 2026-02-20T21:20:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:20:48 2026 2026-02-20T21:20:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:20:48 2026 2026-02-20T21:20:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:20:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:20:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 9036adf4-a5ba-46a3-9950-a98343c2f236 2026-02-20T21:20:49.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:20:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test_demote_image 100 2026-02-20T21:20:49.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:49.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:49.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:20:49.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:20:49.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:20:49.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:49.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:20:49.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test_demote_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:20:49.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:20:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:20:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5555.64 bytes/sec: 22 MiB/s 2026-02-20T21:20:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_demote_image 2026-02-20T21:20:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:20:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test_demote_image stopped 2026-02-20T21:20:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:20:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:20:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:20:49.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:20:49.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:20:50.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test_demote_image stopped 2026-02-20T21:20:50.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:50.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:50.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:20:50.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:20:50.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:20:50.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:20:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-02-20T21:20:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:20:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:20:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:20:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:20:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:20:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:20:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:20:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:20:50.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:20:50.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:20:50.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:20:50.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-20T21:20:50.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:20:50.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:20:50.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:20:50.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:20:50.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:20:50.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:20:50.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:20:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:20:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:20:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:20:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:20:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:20:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:20:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:20:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:20:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:20:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:20:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:20:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:20:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:20:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:20:50.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:20:50.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:20:50.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:20:50.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:20:50.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:20:50.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:20:50.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:20:50.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:20:50.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:20:50.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:20:50.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:20:50.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:20:50.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-20T21:20:50.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-20T21:20:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:20:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:20:50.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:20:50.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_demote_image 2026-02-20T21:20:50.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:20:50.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:20:50.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:20:50.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:20:50.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:20:50.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:20:50.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:20:50.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:20:50.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:20:50.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:20:50.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:20:50.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:20:50.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:20:50.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:20:50.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:20:50.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:20:50.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:20:50.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:20:50.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:20:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:20:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:20:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:20:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:20:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:20:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:20:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:20:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:20:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:20:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:20:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:20:50.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:20:50.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:20:50.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:20:50.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-02-20T21:20:50.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-20T21:20:50.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:20:50.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:20:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:20:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_demote_image 2026-02-20T21:20:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:2 2026-02-20T21:20:50.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:20:50.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:20:50.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-20T21:20:50.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:2 2026-02-20T21:20:50.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:20:50.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:20:50.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:20:50.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:20:50.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-20T21:20:50.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:20:50.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:20:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:20:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:20:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-20T21:20:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:2 2026-02-20T21:20:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:20:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:20:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-20T21:20:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:2 2026-02-20T21:20:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:20:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:20:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:20:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:20:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-20T21:20:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:20:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:20:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:20:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:20:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-20T21:20:50.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:20:50.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:20:50.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:20:50.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_demote_image 2026-02-20T21:20:50.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-20T21:20:50.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:20:50.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:20:50.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:20:50.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_demote_image 2026-02-20T21:20:50.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:3 2026-02-20T21:20:50.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:20:50.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:20:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-20T21:20:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:3 2026-02-20T21:20:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:20:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:20:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:20:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:20:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-20T21:20:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:20:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:20:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:20:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:20:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-02-20T21:20:50.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:3 2026-02-20T21:20:50.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:20:50.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:20:50.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-20T21:20:50.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:3 2026-02-20T21:20:50.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:20:50.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:20:50.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:20:50.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:20:50.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-20T21:20:50.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:20:50.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:20:50.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:20:50.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:20:50.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-20T21:20:50.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:20:50.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:20:50.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:20:50.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_demote_image 2026-02-20T21:20:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-20T21:20:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:20:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:20:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:20:50.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 1 2026-02-20T21:20:51.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:20:51.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:20:51.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:20:51.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-20T21:20:51.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:20:51.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:20:51.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:20:51.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:20:51.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:20:51.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:20:51.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:20:51.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:20:51.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:20:51.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:20:51.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:20:51.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:20:51.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:20:51.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:20:51.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:20:51.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:20:51.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:20:51.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:20:51.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:20:51.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:20:51.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:20:51.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:20:51.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:20:51.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:20:51.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:20:51.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:20:51.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:20:51.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:20:51.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:20:51.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:20:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:20:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:20:51.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:20:51.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-20T21:20:51.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-20T21:20:51.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:20:51.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:20:51.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:20:51.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_demote_image 2026-02-20T21:20:51.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:20:51.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:20:51.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:20:51.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:20:51.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:20:51.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:20:51.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:20:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:20:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:20:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:20:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:20:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:20:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:20:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:20:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:20:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:20:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:20:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:20:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:20:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:20:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:20:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:20:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:20:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:20:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:20:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:20:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:20:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:20:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:20:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:20:51.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:20:51.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:20:51.543 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:20:51.543 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-02-20T21:20:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-20T21:20:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:20:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:20:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:20:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_demote_image 2026-02-20T21:20:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:2 2026-02-20T21:20:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:20:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:20:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-20T21:20:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:2 2026-02-20T21:20:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:20:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:20:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:20:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:20:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-20T21:20:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:20:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:20:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:20:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:20:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-20T21:20:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:2 2026-02-20T21:20:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:20:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:20:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-20T21:20:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:2 2026-02-20T21:20:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:20:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:20:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:20:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:20:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-20T21:20:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:20:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:20:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:20:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:20:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-20T21:20:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:20:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:20:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:20:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_demote_image 2026-02-20T21:20:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:20:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:20:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:20:51.641 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:20:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:20:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test_demote_image 2026-02-20T21:20:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:20:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test_demote_image started 2026-02-20T21:20:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:20:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:20:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:20:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:20:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:20:52.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test_demote_image started 2026-02-20T21:20:52.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:52.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:52.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:20:52.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:20:52.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:20:52.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:20:52.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_demote_image 2026-02-20T21:20:52.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:20:52.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:20:52.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:20:52.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:20:52.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:20:52.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:20:52.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:20:52.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:20:52.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:20:52.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:20:52.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:20:52.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_demote_image 2026-02-20T21:20:52.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:20:52.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:20:52.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:20:52.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:20:52.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:20:52.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:20:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:20:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:20:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:20:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:20:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:20:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:20:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:20:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:20:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:20:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:20:52.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:20:52.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:20:52.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:20:52.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:20:52.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:20:52.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:20:52.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:20:52.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:20:52.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:20:52.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:20:52.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:20:52.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:20:52.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:20:52.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:20:52.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:20:52.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:20:52.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:20:52.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-20T21:20:52.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-20T21:20:52.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:20:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:20:52.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:20:52.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_demote_image 2026-02-20T21:20:52.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:20:52.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:20:52.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:20:52.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:20:52.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:20:52.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:20:52.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:20:52.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:20:52.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:20:52.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:20:52.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:20:52.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:20:52.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:20:52.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:20:52.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:20:52.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:20:52.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:20:52.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:20:52.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:20:52.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:20:52.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:20:52.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:20:52.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:20:52.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:20:52.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:20:52.710 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:20:52.710 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:20:52.710 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:20:52.710 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:20:52.710 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:20:52.718 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:20:52.718 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:20:52.718 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:20:52.718 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-02-20T21:20:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:20:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:20:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:20:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:20:52.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:20:52.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:20:52.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:20:52.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:52.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test_demote_image 2026-02-20T21:20:52.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:20:52.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:52.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:20:52.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:20:52.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:20:52.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:20:52.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:20:52.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test_demote_image 2026-02-20T21:20:52.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:52.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:52.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:20:52.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_demote_image 2026-02-20T21:20:52.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 178 2026-02-20T21:20:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test_demote_image 2026-02-20T21:20:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:20:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:20:52.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:20:52.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:20:52.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test_demote_image.status 2026-02-20T21:20:52.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test_demote_image.status 2026-02-20T21:20:52.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:20:52.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_demote_image.status 2026-02-20T21:20:52.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test_demote_image.status 2026-02-20T21:20:52.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test_demote_image.status 2026-02-20T21:20:52.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:20:52.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:20:52.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_demote_image.status 2026-02-20T21:20:52.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:52.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:52.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:20:52.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_demote_image.status 2026-02-20T21:20:52.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:20:52.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:20:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:20:52.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=178 2026-02-20T21:20:52.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:20:52.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:20:52.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:20:53.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:20:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:20:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:20:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:20:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:20:53.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:20:53.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=177 2026-02-20T21:20:53.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 178 = 177 2026-02-20T21:20:53.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:20:53.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.4 2026-02-20T21:20:53.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:20:53.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:53.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:53.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:20:53.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:20:53.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:20:53.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:20:53.657 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:20:53.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=177 2026-02-20T21:20:53.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 178 = 177 2026-02-20T21:20:53.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:20:53.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.8 2026-02-20T21:20:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:20:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:20:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:20:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:20:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:20:54.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:20:54.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=178 2026-02-20T21:20:54.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 178 = 178 2026-02-20T21:20:54.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:54.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+replaying 2026-02-20T21:20:54.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:54.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:54.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:20:54.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:20:54.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:20:54.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:20:54.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:20:54.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:20:55.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+replaying '' '' 2026-02-20T21:20:55.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:55.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:55.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:20:55.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:20:55.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:20:55.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:20:55.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:20:55.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:20:55.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-02-20T21:20:55.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_demote_image: 2026-02-20T21:20:55.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 9036adf4-a5ba-46a3-9950-a98343c2f236 2026-02-20T21:20:55.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:20:55.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622449,"replay_state":"idle"} 2026-02-20T21:20:55.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:20:55.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:20:51 2026-02-20T21:20:55.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:20:55.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:20:55.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:20:55.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:20:55.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:20:51' 2026-02-20T21:20:55.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:20:55.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:20:55.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:20:55.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622449,"replay_state":"idle"} 2026-02-20T21:20:55.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:20:55.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:20:55.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:20:55.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:20:55.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:20:55.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:20:55.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:20:55.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:20:55.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:20:51' 2026-02-20T21:20:55.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+replaying '2026-02-20 21:20:51' 2026-02-20T21:20:55.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:55.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:55.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:20:55.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:20:55.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:20:51' 2026-02-20T21:20:55.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:20:55.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:20:55.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:20:55.568 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK244044409mirror.0trial19920.2.0-677-gf7870454trueOK44114415mirror.1trial19920.2.0-677-gf7870454falseOK44174425mirror.2trial19920.2.0-677-gf7870454falseOK44274433mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}44044409mirror.0trial1992026-02-20 21:20:28cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:20:51test_demote_image9036adf4-a5ba-46a3-9950-a98343c2f236up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622449,"replay_state":"idle"}44114415mirror.1trial1992026-02-20 21:20:51cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:20:51' 2026-02-20T21:20:55.568 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:20:55.569 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-20T21:20:55.569 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:20:51' 2026-02-20T21:20:55.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-20T21:20:55.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:20:55.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:20:55.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:20:55.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:20:55.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:55.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:55.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+stopped 2026-02-20T21:20:55.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:55.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:55.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:20:55.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:20:55.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:20:55.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:20:55.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:20:55.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:20:56.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+stopped '' '' 2026-02-20T21:20:56.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:56.575 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:56.575 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:20:56.575 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:20:56.575 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:20:56.575 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:20:56.575 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:20:56.575 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:20:56.575 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-02-20T21:20:56.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_demote_image: 2026-02-20T21:20:56.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 9036adf4-a5ba-46a3-9950-a98343c2f236 2026-02-20T21:20:56.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:20:56.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:20:56.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.2 on trial199 2026-02-20T21:20:56.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:20:51 2026-02-20T21:20:56.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:20:56.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:20:56.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:20:56.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622449,"replay_state":"idle"} 2026-02-20T21:20:56.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:20:51 2026-02-20T21:20:56.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:20:56.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 177 .mirror.primary.9036adf4-a5ba-46a3-9950-a98343c2f236.29074e5e-2e5f-484a-80c0-2883da2474f7 (peer_uuids:[]) 2026-02-20T21:20:56.607 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 178 .mirror.primary.9036adf4-a5ba-46a3-9950-a98343c2f236.22014939-6e8d-4707-ba2a-9ce933963433 (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:20:56.607 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:20:56.607 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:20:56.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:20:56.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:20:56.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:20:56.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:20:56.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:20:56.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:20:56.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:20:56.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.2 on trial199 2026-02-20T21:20:56.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:20:56.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:20:56.611 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:20:51' 2026-02-20T21:20:56.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+stopped '2026-02-20 21:20:51' 2026-02-20T21:20:56.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:20:56.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:20:56.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:20:56.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:20:56.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:20:51' 2026-02-20T21:20:56.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:20:56.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:20:56.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:20:56.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK245294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:20:51cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}2026-02-20 21:20:28test_demote_image9036adf4-a5ba-46a3-9950-a98343c2f236up+stoppedlocal image is primary45364539mirror.2trial1992026-02-20 21:20:51cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622449,"replay_state":"idle"}2026-02-20 21:20:51' 2026-02-20T21:20:56.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:20:56.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-20T21:20:56.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:20:51' 2026-02-20T21:20:56.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-20T21:20:56.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:20:56.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:20:56.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:20:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:20:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:20:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirrors cluster1 2026-02-20T21:20:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:20:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:20:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:20:56.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:20:56.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:0 '' 2026-02-20T21:20:56.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:20:56.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:20:56.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:56.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:20:56.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:0 2026-02-20T21:20:56.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:0 2026-02-20T21:20:56.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:20:56.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:20:56.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:20:56.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:20:56.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:20:56.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:20:56.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:20:56.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:20:56.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:20:56.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:20:56.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:20:56.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:20:56.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-20T21:20:56.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:20:56.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:20:56.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=50691 2026-02-20T21:20:56.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 50691 ']' 2026-02-20T21:20:56.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 50691 2026-02-20T21:20:56.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:20:56.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:20:56.666 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:20:56.666 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=50691 '$2 == pid {print; exit 1}' 2026-02-20T21:20:56.680 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:ubuntu 50691 0.2 0.0 552876 73444 ? Ssl 21:04 0:02 rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:20:56.680 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:20:56.680 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:20:57.682 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:20:57.682 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=50691 '$2 == pid {print; exit 1}' 2026-02-20T21:20:57.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:20:57.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:20:57.695 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=50691 '$2 == pid {print; exit 1}' 2026-02-20T21:20:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:20:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:20:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:20:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:20:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:20:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:20:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:20:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:20:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:20:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:20:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:20:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:20:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:20:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:20:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:20:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:20:57.719 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:20:57.719 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:20:57.721 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-20T21:20:57.721 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:20:57.721 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:20:57.721 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:20:57.721 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:20:57.721 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:20:57.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:20:57.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:20:57.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:20:57.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:20:57.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:20:57.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:20:57.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:20:57.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:20:57.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:20:57.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:20:57.733 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:20:57.734 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:20:57.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:0 2026-02-20T21:20:57.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:20:57.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:20:57.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:20:57.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:20:57.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:20:57.737 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:20:57.737 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:20:57.737 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:20:57.737 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:20:57.737 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:20:57.737 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:20:57.737 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:20:57.737 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:20:57.737 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-20T21:20:57.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:20:57.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:20:57.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:20:57.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:1 '' 2026-02-20T21:20:57.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:20:57.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:20:57.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:57.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:20:57.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:1 2026-02-20T21:20:57.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:1 2026-02-20T21:20:57.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:20:57.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:20:57.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:20:57.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:20:57.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:20:57.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:20:57.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:20:57.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:20:57.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:20:57.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:20:57.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:20:57.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:20:57.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-20T21:20:57.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:20:57.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:20:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=50734 2026-02-20T21:20:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 50734 ']' 2026-02-20T21:20:57.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 50734 2026-02-20T21:20:57.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:20:57.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:20:57.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:20:57.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=50734 '$2 == pid {print; exit 1}' 2026-02-20T21:20:57.776 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:ubuntu 50734 0.1 0.0 540580 69828 ? Ssl 21:04 0:01 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:20:57.776 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:20:57.777 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:20:58.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:20:58.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=50734 '$2 == pid {print; exit 1}' 2026-02-20T21:20:58.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:20:58.791 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:20:58.791 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=50734 '$2 == pid {print; exit 1}' 2026-02-20T21:20:58.802 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:20:58.802 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:20:58.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:20:58.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:20:58.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:20:58.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:20:58.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:20:58.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:20:58.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:20:58.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:20:58.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:20:58.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:20:58.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:20:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:20:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:20:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:20:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:20:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:20:58.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-20T21:20:58.816 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:20:58.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:20:58.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:20:58.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:20:58.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:20:58.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:20:58.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:20:58.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:20:58.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:20:58.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:20:58.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:20:58.818 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:20:58.818 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:20:58.818 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:20:58.818 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:20:58.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:20:58.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:20:58.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:1 2026-02-20T21:20:58.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:20:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:20:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:20:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:20:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:20:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:20:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:20:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:20:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:20:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:20:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:20:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:20:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:20:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-20T21:20:58.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:20:58.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:20:58.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:20:58.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:2 '' 2026-02-20T21:20:58.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:20:58.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:20:58.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:58.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:20:58.843 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:2 2026-02-20T21:20:58.843 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:2 2026-02-20T21:20:58.843 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:20:58.843 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:20:58.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:20:58.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:20:58.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:20:58.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:20:58.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:20:58.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:20:58.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:20:58.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:20:58.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:20:58.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:20:58.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-20T21:20:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:20:58.855 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:20:58.856 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=50796 2026-02-20T21:20:58.856 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 50796 ']' 2026-02-20T21:20:58.856 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 50796 2026-02-20T21:20:58.856 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:20:58.856 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:20:58.857 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:20:58.857 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=50796 '$2 == pid {print; exit 1}' 2026-02-20T21:20:58.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:ubuntu 50796 0.0 0.0 475020 62376 ? Rsl 21:04 0:00 rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:20:58.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:20:58.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:20:59.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:20:59.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=50796 '$2 == pid {print; exit 1}' 2026-02-20T21:20:59.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:20:59.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:20:59.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=50796 '$2 == pid {print; exit 1}' 2026-02-20T21:20:59.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:20:59.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:20:59.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:20:59.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:20:59.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:20:59.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:20:59.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:20:59.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:20:59.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:20:59.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:20:59.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:20:59.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:20:59.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:20:59.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:20:59.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:20:59.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:20:59.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:20:59.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:20:59.905 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-20T21:20:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:20:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:20:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:20:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:20:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:20:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:20:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:20:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:20:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:20:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:20:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:20:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:20:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:20:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:20:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:20:59.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:20:59.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:20:59.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:2 2026-02-20T21:20:59.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:20:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:20:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:20:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:20:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:20:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:20:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:20:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:20:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:20:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:20:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:20:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:20:59.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:20:59.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-20T21:20:59.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:20:59.929 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:20:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:20:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:3 '' 2026-02-20T21:20:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:20:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:20:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:20:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:20:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:3 2026-02-20T21:20:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:3 2026-02-20T21:20:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:20:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:20:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:20:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:20:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:20:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:20:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:20:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:20:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:20:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:20:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:20:59.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:20:59.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-20T21:20:59.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:20:59.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:20:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=50864 2026-02-20T21:20:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 50864 ']' 2026-02-20T21:20:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 50864 2026-02-20T21:20:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:20:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:20:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:20:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=50864 '$2 == pid {print; exit 1}' 2026-02-20T21:20:59.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:ubuntu 50864 0.0 0.0 597960 63608 ? Ssl 21:04 0:00 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:20:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:20:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:00.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:21:00.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=50864 '$2 == pid {print; exit 1}' 2026-02-20T21:21:00.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:21:00.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:21:00.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=50864 '$2 == pid {print; exit 1}' 2026-02-20T21:21:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:21:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:21:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:21:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:21:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:21:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:21:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:21:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:21:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:21:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:21:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:21:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:21:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:21:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:21:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:21:00.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-20T21:21:00.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:21:00.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:21:00.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:21:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:21:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:21:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:21:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:21:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:21:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:21:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:21:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:21:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:21:01.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:21:01.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:21:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:3 2026-02-20T21:21:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:21:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:21:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:21:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:21:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:21:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:21:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:21:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:21:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:21:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-20T21:21:01.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:21:01.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:21:01.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test_demote_image 100 2026-02-20T21:21:01.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:01.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:01.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:01.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:21:01.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:21:01.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:21:01.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:21:01.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test_demote_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:21:01.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:21:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:21:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5263.24 bytes/sec: 21 MiB/s 2026-02-20T21:21:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test_demote_image 2026-02-20T21:21:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test_demote_image 2026-02-20T21:21:02.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:21:02.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirrors cluster1 2026-02-20T21:21:02.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:02.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:21:02.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:02.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:0 2026-02-20T21:21:02.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:21:02.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:21:02.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:21:02.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:21:02.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:21:02.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:21:02.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:21:02.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:21:02.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:21:02.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:21:02.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:02.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:21:02.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:21:02.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:21:02.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:21:02.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:02.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:1 2026-02-20T21:21:02.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:21:02.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:21:02.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:21:02.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:21:02.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:21:02.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:21:02.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:21:02.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:21:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:21:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:21:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:21:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:21:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:21:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:21:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:2 2026-02-20T21:21:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:21:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:21:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:21:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:2 2026-02-20T21:21:03.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:21:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:21:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:21:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=2 2026-02-20T21:21:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:21:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:21:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=2 2026-02-20T21:21:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:21:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:21:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:21:03.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:03.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:3 2026-02-20T21:21:03.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:21:03.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:21:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:21:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:3 2026-02-20T21:21:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:21:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:21:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:21:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=3 2026-02-20T21:21:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:21:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:21:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=3 2026-02-20T21:21:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:21:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:21:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:21:03.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test_demote_image 2026-02-20T21:21:03.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:21:03.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:03.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:21:03.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:21:03.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:03.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test_demote_image.status 2026-02-20T21:21:03.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test_demote_image.status 2026-02-20T21:21:03.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:21:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_demote_image.status 2026-02-20T21:21:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test_demote_image.status 2026-02-20T21:21:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test_demote_image.status 2026-02-20T21:21:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:21:03.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:21:03.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_demote_image.status 2026-02-20T21:21:03.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:03.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:03.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:03.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_demote_image.status 2026-02-20T21:21:03.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:21:03.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:21:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:21:03.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=179 2026-02-20T21:21:03.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:21:03.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:21:03.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:21:03.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:21:03.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:03.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:03.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:03.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:21:03.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:21:03.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:21:03.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:21:03.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=178 2026-02-20T21:21:03.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 179 = 178 2026-02-20T21:21:03.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:21:03.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.4 2026-02-20T21:21:03.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:21:03.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:03.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:03.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:03.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:21:03.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:21:03.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:21:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:21:03.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=178 2026-02-20T21:21:03.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 179 = 178 2026-02-20T21:21:03.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:21:03.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.8 2026-02-20T21:21:04.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:21:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:21:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:21:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:21:04.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:21:04.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=178 2026-02-20T21:21:04.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 179 = 178 2026-02-20T21:21:04.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:21:04.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1.6 2026-02-20T21:21:06.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:21:06.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:06.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:06.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:06.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:21:06.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:21:06.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:21:06.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:21:06.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=178 2026-02-20T21:21:06.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 179 = 178 2026-02-20T21:21:06.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:21:06.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:21:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:21:08.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:08.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:08.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:08.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:21:08.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:21:08.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:21:08.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:21:08.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=179 2026-02-20T21:21:08.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 179 = 179 2026-02-20T21:21:08.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:08.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+unknown 2026-02-20T21:21:08.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:08.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:08.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:08.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:21:08.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:21:08.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:08.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:08.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:09.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+unknown '' '' 2026-02-20T21:21:09.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:09.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:09.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:09.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:21:09.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:21:09.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:09.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:09.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:09.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-02-20T21:21:09.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_demote_image: 2026-02-20T21:21:09.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 9036adf4-a5ba-46a3-9950-a98343c2f236 2026-02-20T21:21:09.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:21:09.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:21:09.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.2 on trial199 2026-02-20T21:21:09.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:06 2026-02-20T21:21:09.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:21:09.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:21:09.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:21:09.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:21:09.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:08' 2026-02-20T21:21:09.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:21:09.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:21:09.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:21:09.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:21:09.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:21:09.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:21:09.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:21:09.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:21:09.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:21:09.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.2 on trial199 2026-02-20T21:21:09.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:21:09.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:21:09.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:06' 2026-02-20T21:21:09.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+unknown '2026-02-20 21:21:06' 2026-02-20T21:21:09.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:09.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:09.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:09.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:21:09.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:21:06' 2026-02-20T21:21:09.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:09.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:09.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:21:09.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING1145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:21:06cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}2026-02-20 21:21:07test_demote_image9036adf4-a5ba-46a3-9950-a98343c2f236up+unknownremote image is not primary45364539mirror.2trial1992026-02-20 21:21:06cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image demoted2026-02-20 21:21:08' 2026-02-20T21:21:09.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:21:09.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-20T21:21:09.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:06' 2026-02-20T21:21:09.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-20T21:21:09.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:21:09.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:21:09.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:21:09.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:21:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+unknown 2026-02-20T21:21:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:21:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:21:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:09.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+unknown '' '' 2026-02-20T21:21:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:21:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:21:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:10.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:10.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:10.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-02-20T21:21:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_demote_image: 2026-02-20T21:21:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 9036adf4-a5ba-46a3-9950-a98343c2f236 2026-02-20T21:21:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:21:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:21:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:21:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:08 2026-02-20T21:21:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:21:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:21:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:21:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:21:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:06' 2026-02-20T21:21:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:21:10.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:21:10.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:21:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image demoted 2026-02-20T21:21:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:21:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:21:10.376 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:21:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:21:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:21:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:21:10.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:21:10.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:21:10.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:08' 2026-02-20T21:21:10.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+unknown '2026-02-20 21:21:08' 2026-02-20T21:21:10.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:10.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:10.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:10.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:21:10.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:21:08' 2026-02-20T21:21:10.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:10.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:10.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:21:10.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING1156715675mirror.0trial19920.2.0-677-gf7870454trueOK56775682mirror.1trial19920.2.0-677-gf7870454falseOK56845690mirror.2trial19920.2.0-677-gf7870454falseOK56925697mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}56715675mirror.0trial1992026-02-20 21:21:07cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:21:06test_demote_image9036adf4-a5ba-46a3-9950-a98343c2f236up+unknownremote image demoted56775682mirror.1trial1992026-02-20 21:21:08cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:21:06' 2026-02-20T21:21:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:21:10.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-20T21:21:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:08' 2026-02-20T21:21:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-20T21:21:10.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:21:10.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:21:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:21:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:21:10.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror mirror test_demote_image 2026-02-20T21:21:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:21:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:21:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:21:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:21:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test_demote_image.export 2026-02-20T21:21:10.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test_demote_image.export 2026-02-20T21:21:10.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:21:10.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_demote_image.export 2026-02-20T21:21:10.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test_demote_image.export 2026-02-20T21:21:10.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test_demote_image.export 2026-02-20T21:21:10.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:21:10.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.export 2026-02-20T21:21:10.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_demote_image.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.export 2026-02-20T21:21:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/test_demote_image /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_demote_image.export 2026-02-20T21:21:10.507 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:21:10.512 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.export 2026-02-20T21:21:10.602 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:21:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_demote_image.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.export 2026-02-20T21:21:10.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_demote_image.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.export 2026-02-20T21:21:10.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:10.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test_demote_image 2026-02-20T21:21:10.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:10.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:10.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:10.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:21:10.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_demote_image 2026-02-20T21:21:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:21:11.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test_demote_image 2026-02-20T21:21:11.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:11.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:11.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:11.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test_demote_image started 2026-02-20T21:21:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:21:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:21:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:12.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test_demote_image started 2026-02-20T21:21:12.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:12.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:12.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:12.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:21:12.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:21:12.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:21:12.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-02-20T21:21:12.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:21:12.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:12.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:21:12.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:21:12.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:21:12.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:12.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:21:12.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:21:12.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:21:12.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:21:12.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:12.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-20T21:21:12.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:21:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:21:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:21:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:21:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:21:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:21:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:21:12.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:12.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:21:12.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:21:12.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:21:12.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:21:12.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:21:12.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:12.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:21:12.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:21:12.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:12.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:12.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:12.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:21:12.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:21:12.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:21:12.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:21:12.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:21:12.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:21:12.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:21:12.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:21:12.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:21:12.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:21:12.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-20T21:21:12.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-20T21:21:12.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:21:12.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:21:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_demote_image 2026-02-20T21:21:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:21:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:12.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:12.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:21:12.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:21:12.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:12.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:12.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:21:12.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:21:12.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:21:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:21:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:21:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:21:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:21:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:21:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:21:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:21:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:21:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:12.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:12.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:21:12.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:21:12.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:21:12.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:21:12.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:21:12.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:21:12.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:21:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:21:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:21:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:21:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-02-20T21:21:12.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-20T21:21:12.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:21:12.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:21:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_demote_image 2026-02-20T21:21:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:2 2026-02-20T21:21:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-20T21:21:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:2 2026-02-20T21:21:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:21:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:21:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-20T21:21:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:21:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:21:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:21:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-20T21:21:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:2 2026-02-20T21:21:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:21:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-20T21:21:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:2 2026-02-20T21:21:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:21:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-20T21:21:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:21:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:21:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:21:12.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:21:12.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-20T21:21:12.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:21:12.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:21:12.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:21:12.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_demote_image 2026-02-20T21:21:12.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:21:12.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:21:12.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:21:12.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:21:12.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:21:12.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:21:12.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:21:12.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:12.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+stopped 2026-02-20T21:21:12.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:21:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:21:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:13.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+stopped '' '' 2026-02-20T21:21:13.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:13.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:13.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:13.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:21:13.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:21:13.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:13.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:13.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:13.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-02-20T21:21:13.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_demote_image: 2026-02-20T21:21:13.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 9036adf4-a5ba-46a3-9950-a98343c2f236 2026-02-20T21:21:13.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:21:13.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:21:13.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:21:13.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:13 2026-02-20T21:21:13.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:21:13.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:21:13.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:21:13.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622470,"replay_state":"idle"} 2026-02-20T21:21:13.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:11 2026-02-20T21:21:13.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:21:13.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 185 .mirror.primary.9036adf4-a5ba-46a3-9950-a98343c2f236.76347189-6f34-4c17-ae76-a421f190384b (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:21:13.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:21:13.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:21:13.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:21:13.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:21:13.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:21:13.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:21:13.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:21:13.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:21:13.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:21:13.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:21:13.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:21:13.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:21:13.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:13' 2026-02-20T21:21:13.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+stopped '2026-02-20 21:21:13' 2026-02-20T21:21:13.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:13.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:13.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:13.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:21:13.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:21:13' 2026-02-20T21:21:13.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:13.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:13.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:21:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK256715675mirror.0trial19920.2.0-677-gf7870454trueOK56775682mirror.1trial19920.2.0-677-gf7870454falseOK56845690mirror.2trial19920.2.0-677-gf7870454falseOK56925697mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}56715675mirror.0trial1992026-02-20 21:21:07cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:21:11test_demote_image9036adf4-a5ba-46a3-9950-a98343c2f236up+stoppedlocal image is primary56775682mirror.1trial1992026-02-20 21:21:13cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622470,"replay_state":"idle"}2026-02-20 21:21:11' 2026-02-20T21:21:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:21:13.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-20T21:21:13.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:13' 2026-02-20T21:21:13.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-20T21:21:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:21:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:21:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:21:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:21:13.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:13.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:13.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+replaying 2026-02-20T21:21:13.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:13.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:21:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:21:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:14.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+replaying '' '' 2026-02-20T21:21:14.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:14.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:14.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:14.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:21:14.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:21:14.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:14.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:14.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:14.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-02-20T21:21:14.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_demote_image: 2026-02-20T21:21:14.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 9036adf4-a5ba-46a3-9950-a98343c2f236 2026-02-20T21:21:14.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:21:14.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622470,"replay_state":"idle"} 2026-02-20T21:21:14.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.2 on trial199 2026-02-20T21:21:14.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:11 2026-02-20T21:21:14.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:21:14.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:21:14.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:21:14.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:21:14.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:13' 2026-02-20T21:21:14.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:21:14.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:21:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:21:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622470,"replay_state":"idle"} 2026-02-20T21:21:14.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:21:14.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:21:14.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:21:14.499 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:21:14.499 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:21:14.499 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.2 on trial199 2026-02-20T21:21:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:21:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:21:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:11' 2026-02-20T21:21:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+replaying '2026-02-20 21:21:11' 2026-02-20T21:21:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:21:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:21:11' 2026-02-20T21:21:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:14.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:21:14.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK245294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:21:11cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}2026-02-20 21:21:07test_demote_image9036adf4-a5ba-46a3-9950-a98343c2f236up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622470,"replay_state":"idle"}45364539mirror.2trial1992026-02-20 21:21:11cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:21:13' 2026-02-20T21:21:14.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:21:14.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-20T21:21:14.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:11' 2026-02-20T21:21:14.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-20T21:21:14.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:21:14.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:21:14.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:21:14.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:21:14.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:14.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:14.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster1 mirror test_demote_image 2026-02-20T21:21:14.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:14.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:14.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:14.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:21:14.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:21:14.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster1 mirror test_demote_image 2026-02-20T21:21:14.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:14.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:14.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:21:14.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_demote_image 2026-02-20T21:21:14.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 rm mirror/test_demote_image 2026-02-20T21:21:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:21:16.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:16.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:21:16.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: request image resync when remote is not primary' 2026-02-20T21:21:16.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:21:16.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:21:16.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:21:16 TEST: request image resync when remote is not primary 2026-02-20T21:21:16.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:21:16 TEST: request image resync when remote is not primary 2026-02-20T21:21:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_resync_image=test_resync_image 2026-02-20T21:21:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image_and_enable_mirror cluster2 mirror test_resync_image snapshot 2026-02-20T21:21:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:21:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:21:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:21:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:21:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:21:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:21:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image cluster2 mirror test_resync_image 2026-02-20T21:21:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:21:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:21:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:21:16.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=128 2026-02-20T21:21:16.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:21:16.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,object-map,fast-diff mirror/test_resync_image 2026-02-20T21:21:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:21:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:21:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror test_resync_image snapshot 2026-02-20T21:21:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:21:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_resync_image snapshot 2026-02-20T21:21:16.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:21:16.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/test_resync_image 2026-02-20T21:21:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test_resync_image': 2026-02-20T21:21:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:21:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:21:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:21:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 176371c1ba6c 2026-02-20T21:21:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.176371c1ba6c 2026-02-20T21:21:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:21:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:21:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:21:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:21:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:21:16 2026 2026-02-20T21:21:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:21:16 2026 2026-02-20T21:21:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:21:16 2026 2026-02-20T21:21:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:21:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:21:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 9b0ec88c-a9ab-46dd-9407-72e81a713c20 2026-02-20T21:21:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:21:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test_resync_image 100 2026-02-20T21:21:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:21:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:21:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:21:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:21:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test_resync_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:21:17.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:21:17.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:21:17.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5263.24 bytes/sec: 21 MiB/s 2026-02-20T21:21:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_resync_image 2026-02-20T21:21:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test_resync_image stopped 2026-02-20T21:21:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:21:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:21:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test_resync_image stopped 2026-02-20T21:21:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:21:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:21:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:21:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:21:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:21:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:21:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:21:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:21:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:21:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:21:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:21:18.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:18.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:18.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:21:18.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:18.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:18.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:21:18.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:21:18.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:18.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:18.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:21:18.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:21:18.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:21:18.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:21:18.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:18.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:21:18.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:21:18.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:21:18.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:21:18.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:21:18.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:21:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:21:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:21:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:21:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:21:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:21:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:21:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:21:18.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:21:18.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:21:18.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:21:18.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:21:18.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:18.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-20T21:21:18.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:21:18.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:21:18.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:18.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:18.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:21:18.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:18.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:18.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:21:18.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:21:18.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:18.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:18.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:21:18.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:21:18.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:21:18.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:21:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:21:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:21:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:21:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:21:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:21:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:21:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:21:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:21:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:21:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:21:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:21:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:21:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:21:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:21:18.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:21:18.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:21:18.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:21:18.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:18.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-20T21:21:18.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:21:18.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:2 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:2 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:2 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:2 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:21:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:21:18.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-20T21:21:18.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:21:18.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:21:18.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:21:18.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:18.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-20T21:21:18.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:21:18.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:21:18.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:18.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:18.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:3 2026-02-20T21:21:18.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:18.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:18.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-20T21:21:18.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:3 2026-02-20T21:21:18.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:18.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:18.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:21:18.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:21:18.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-20T21:21:18.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:21:18.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:18.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:21:18.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:21:18.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-02-20T21:21:18.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:3 2026-02-20T21:21:18.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:21:18.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:18.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-20T21:21:18.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:3 2026-02-20T21:21:18.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:18.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:18.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:18.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:21:18.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-20T21:21:18.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:21:18.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:21:18.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:21:18.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:21:18.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-20T21:21:18.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:21:18.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:21:18.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:21:18.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:18.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-20T21:21:18.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:21:18.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:21:18.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:18.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 1 2026-02-20T21:21:19.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:21:19.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:21:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:21:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:21:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:21:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:21:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:21:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:21:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:21:19.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:19.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:21:19.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:21:19.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:21:19.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:21:19.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:21:19.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:19.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:21:19.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:21:19.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:21:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:21:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:21:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:21:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:21:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:21:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:21:19.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:21:19.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:21:19.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:21:19.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-20T21:21:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:21:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:21:19.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:19.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:19.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:21:19.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:19.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:19.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:21:19.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:21:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:21:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:21:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:21:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:21:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:21:19.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:21:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:21:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:21:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:21:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:21:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:21:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:21:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:21:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:21:19.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:21:19.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:21:19.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:21:19.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:21:19.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:21:19.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:21:19.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:21:19.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:19.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-20T21:21:19.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:21:19.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:21:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:2 2026-02-20T21:21:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-20T21:21:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:2 2026-02-20T21:21:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:21:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:21:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-20T21:21:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:21:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:21:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:21:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-20T21:21:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:2 2026-02-20T21:21:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:21:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-20T21:21:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:2 2026-02-20T21:21:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:21:19.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-20T21:21:19.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:21:19.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:21:19.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:21:19.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:21:19.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-20T21:21:19.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:21:19.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:21:19.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:21:19.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:19.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:21:19.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:21:19.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:21:19.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:21:19.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:21:19.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:19.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test_resync_image 2026-02-20T21:21:19.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:19.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:19.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:19.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test_resync_image started 2026-02-20T21:21:19.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:19.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:19.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:19.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:21:19.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:21:19.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:19.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:20.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test_resync_image started 2026-02-20T21:21:20.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:20.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:20.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:20.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:21:20.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:21:20.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:21:20.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:20.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:21:20.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:21:20.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:20.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:21:20.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:21:20.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:21:20.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:20.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:21:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:21:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:21:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:21:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:21:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:21:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:20.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:21:20.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:21:20.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:21:20.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:21:20.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:21:20.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:21:20.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:21:20.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:21:20.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:21:20.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:20.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:21:20.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:21:20.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:20.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:20.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:21:20.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:21:20.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:21:20.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:21:20.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:21:20.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:21:20.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:21:20.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:21:20.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:21:20.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:21:20.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:21:20.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:20.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-20T21:21:20.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:21:20.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:21:20.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:20.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:20.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:21:20.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:20.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:20.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:21:20.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:21:20.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:20.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:20.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:20.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:21:20.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:21:20.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:21:20.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:21:20.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:21:20.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:21:20.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:21:20.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:21:20.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:21:20.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:20.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:21:20.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:21:20.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:20.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:20.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:21:20.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:21:20.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:21:20.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:21:20.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:21:20.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:21:20.540 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:21:20.541 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:21:20.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:21:20.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:21:20.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:21:20.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:21:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:21:20.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:21:20.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:21:20.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:21:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:21:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:21:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test_resync_image 2026-02-20T21:21:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:21:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:21:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:21:20.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:21:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:21:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test_resync_image 2026-02-20T21:21:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_resync_image 2026-02-20T21:21:20.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 182 2026-02-20T21:21:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test_resync_image 2026-02-20T21:21:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:21:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:21:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:21:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test_resync_image.status 2026-02-20T21:21:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test_resync_image.status 2026-02-20T21:21:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:21:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_resync_image.status 2026-02-20T21:21:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test_resync_image.status 2026-02-20T21:21:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test_resync_image.status 2026-02-20T21:21:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:21:21.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.status 2026-02-20T21:21:21.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_resync_image /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_resync_image.status 2026-02-20T21:21:21.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:21.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:21.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:21.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_resync_image.status 2026-02-20T21:21:21.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_resync_image --format xml 2026-02-20T21:21:21.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:21:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:21:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=182 2026-02-20T21:21:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:21:21.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:21:21.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:21:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_resync_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.status 2026-02-20T21:21:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.status 2026-02-20T21:21:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_resync_image --format xml 2026-02-20T21:21:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:21:21.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:21:21.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=181 2026-02-20T21:21:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 182 = 181 2026-02-20T21:21:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:21:21.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.4 2026-02-20T21:21:21.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_resync_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.status 2026-02-20T21:21:21.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:21.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:21.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:21.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.status 2026-02-20T21:21:21.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_resync_image --format xml 2026-02-20T21:21:21.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:21:21.697 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:21:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=181 2026-02-20T21:21:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 182 = 181 2026-02-20T21:21:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:21:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.8 2026-02-20T21:21:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_resync_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.status 2026-02-20T21:21:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.status 2026-02-20T21:21:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_resync_image --format xml 2026-02-20T21:21:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:21:22.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:21:22.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=182 2026-02-20T21:21:22.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 182 = 182 2026-02-20T21:21:22.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:22.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_resync_image 2026-02-20T21:21:22.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:22.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:22.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:22.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:21:22.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+replaying 2026-02-20T21:21:22.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:21:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:21:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:23.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+replaying '' '' 2026-02-20T21:21:23.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:21:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:21:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-20T21:21:23.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_resync_image: 2026-02-20T21:21:23.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 9b0ec88c-a9ab-46dd-9407-72e81a713c20 2026-02-20T21:21:23.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:21:23.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622476,"replay_state":"idle"} 2026-02-20T21:21:23.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:21:23.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:19 2026-02-20T21:21:23.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:21:23.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:21:23.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:21:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:21:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:21' 2026-02-20T21:21:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:21:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:21:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:21:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622476,"replay_state":"idle"} 2026-02-20T21:21:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:21:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:21:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:21:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:21:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:21:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:21:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:21:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:21:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:19' 2026-02-20T21:21:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+replaying '2026-02-20 21:21:19' 2026-02-20T21:21:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:21:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:21:19' 2026-02-20T21:21:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:21:23.814 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK256715675mirror.0trial19920.2.0-677-gf7870454trueOK56775682mirror.1trial19920.2.0-677-gf7870454falseOK56845690mirror.2trial19920.2.0-677-gf7870454falseOK56925697mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}56715675mirror.0trial1992026-02-20 21:21:07cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:21:21test_resync_image9b0ec88c-a9ab-46dd-9407-72e81a713c20up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622476,"replay_state":"idle"}56775682mirror.1trial1992026-02-20 21:21:19cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:21:21' 2026-02-20T21:21:23.814 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:21:23.814 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-20T21:21:23.815 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:19' 2026-02-20T21:21:23.815 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-20T21:21:23.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:21:23.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:21:23.817 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:21:23.818 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:21:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+stopped 2026-02-20T21:21:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:23.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:21:23.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:21:23.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:23.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:23.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:24.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+stopped '' '' 2026-02-20T21:21:24.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:24.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:24.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:24.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:21:24.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:21:24.821 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:24.821 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:24.821 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:24.821 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-02-20T21:21:24.851 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_resync_image: 2026-02-20T21:21:24.851 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 9b0ec88c-a9ab-46dd-9407-72e81a713c20 2026-02-20T21:21:24.851 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:21:24.851 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:21:24.851 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.2 on trial199 2026-02-20T21:21:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:21 2026-02-20T21:21:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:21:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:21:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:21:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622476,"replay_state":"idle"} 2026-02-20T21:21:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:19 2026-02-20T21:21:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:21:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 181 .mirror.primary.9b0ec88c-a9ab-46dd-9407-72e81a713c20.62fde68e-c66f-4b54-903d-98222abb2734 (peer_uuids:[]) 2026-02-20T21:21:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 182 .mirror.primary.9b0ec88c-a9ab-46dd-9407-72e81a713c20.2c115f33-b321-49d7-9859-88c23aec2e8d (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:21:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:21:24.853 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:21:24.853 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:21:24.854 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:21:24.854 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:21:24.854 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:21:24.855 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:21:24.855 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:21:24.855 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:21:24.856 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.2 on trial199 2026-02-20T21:21:24.856 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:21:24.856 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:21:24.857 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:21' 2026-02-20T21:21:24.857 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+stopped '2026-02-20 21:21:21' 2026-02-20T21:21:24.858 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:24.858 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:24.858 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:24.858 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:21:24.858 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:21:21' 2026-02-20T21:21:24.858 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:24.858 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:24.858 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:21:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK245294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:21:21cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}2026-02-20 21:21:07test_resync_image9b0ec88c-a9ab-46dd-9407-72e81a713c20up+stoppedlocal image is primary45364539mirror.2trial1992026-02-20 21:21:21cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622476,"replay_state":"idle"}2026-02-20 21:21:19' 2026-02-20T21:21:24.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:21:24.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-20T21:21:24.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:21' 2026-02-20T21:21:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-20T21:21:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:21:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:21:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:21:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:21:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test_resync_image 100 2026-02-20T21:21:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:21:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:21:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:21:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:21:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test_resync_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:21:24.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:21:24.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:21:24.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 4545.52 bytes/sec: 18 MiB/s 2026-02-20T21:21:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test_resync_image 2026-02-20T21:21:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:24.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test_resync_image 2026-02-20T21:21:26.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:21:26.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ request_resync_image cluster1 mirror test_resync_image test_resync_image_id 2026-02-20T21:21:26.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:26.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:26.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:26.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id_var_name=test_resync_image_id 2026-02-20T21:21:26.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test_resync_image 2026-02-20T21:21:26.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:21:26.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:21:26.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test_resync_image 2026-02-20T21:21:26.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test_resync_image 2026-02-20T21:21:26.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:21:26.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval 'test_resync_image_id='\''1632f982edc7'\''' 2026-02-20T21:21:26.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test_resync_image_id=1632f982edc7 2026-02-20T21:21:26.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval 'test -n "$test_resync_image_id"' 2026-02-20T21:21:26.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -n 1632f982edc7 2026-02-20T21:21:26.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/test_resync_image 2026-02-20T21:21:26.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Flagged image for resync from primary 2026-02-20T21:21:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+unknown 'remote image is not primary' 2026-02-20T21:21:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:21:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-20T21:21:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:27.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-20T21:21:27.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:27.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:27.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:27.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:21:27.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-20T21:21:27.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:27.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:27.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:27.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-02-20T21:21:27.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_resync_image: 2026-02-20T21:21:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 9b0ec88c-a9ab-46dd-9407-72e81a713c20 2026-02-20T21:21:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:21:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:21:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.2 on trial199 2026-02-20T21:21:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:26 2026-02-20T21:21:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:21:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:21:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:21:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:21:27.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:26' 2026-02-20T21:21:27.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:21:27.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:21:27.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*remote image is not primary' 2026-02-20T21:21:27.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:21:27.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:21:27.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:21:27.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:21:27.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:21:27.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:21:27.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.2 on trial199 2026-02-20T21:21:27.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:21:27.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:21:27.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:26' 2026-02-20T21:21:27.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+unknown '2026-02-20 21:21:26' 2026-02-20T21:21:27.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:27.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:27.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:27.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:21:27.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:21:26' 2026-02-20T21:21:27.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:27.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:27.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:21:27.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING1145294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:21:26cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}2026-02-20 21:21:07test_resync_image9b0ec88c-a9ab-46dd-9407-72e81a713c20up+unknownremote image is not primary45364539mirror.2trial1992026-02-20 21:21:26cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image demoted2026-02-20 21:21:26' 2026-02-20T21:21:27.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:21:27.291 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-20T21:21:27.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:26' 2026-02-20T21:21:27.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-20T21:21:27.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:21:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:21:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:21:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:21:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' 2026-02-20T21:21:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:21:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-20T21:21:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:28.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-20T21:21:28.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:28.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:28.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:28.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:21:28.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-20T21:21:28.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:28.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:28.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:28.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-20T21:21:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_resync_image: 2026-02-20T21:21:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 9b0ec88c-a9ab-46dd-9407-72e81a713c20 2026-02-20T21:21:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:21:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:21:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:21:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:28 2026-02-20T21:21:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:21:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:21:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:21:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:21:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:26' 2026-02-20T21:21:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:21:28.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:21:28.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*remote image is not primary' 2026-02-20T21:21:28.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:21:28.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:21:28.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:21:28.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:21:28.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:21:28.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:21:28.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:21:28.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:21:28.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:21:28.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:28' 2026-02-20T21:21:28.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+unknown '2026-02-20 21:21:28' 2026-02-20T21:21:28.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:28.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:28.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:28.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:21:28.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:21:28' 2026-02-20T21:21:28.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:28.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:28.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:21:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING1156715675mirror.0trial19920.2.0-677-gf7870454trueOK56775682mirror.1trial19920.2.0-677-gf7870454falseOK56845690mirror.2trial19920.2.0-677-gf7870454falseOK56925697mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}56715675mirror.0trial1992026-02-20 21:21:07cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:21:26test_resync_image9b0ec88c-a9ab-46dd-9407-72e81a713c20up+unknownremote image is not primary56775682mirror.1trial1992026-02-20 21:21:28cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:21:26' 2026-02-20T21:21:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:21:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-20T21:21:28.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:28' 2026-02-20T21:21:28.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-20T21:21:28.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:21:28.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:21:28.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:21:28.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:21:28.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:28.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:28.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test_resync_image 2026-02-20T21:21:28.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:28.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:28.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:28.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:21:28.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_resync_image 2026-02-20T21:21:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:21:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test_resync_image 2026-02-20T21:21:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test_resync_image started 2026-02-20T21:21:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:28.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:28.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:21:28.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:21:28.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:28.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test_resync_image started 2026-02-20T21:21:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:21:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:21:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:21:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:21:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:21:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:21:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:21:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:21:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:21:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:21:29.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:21:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:21:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:21:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:21:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:21:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:21:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:21:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:21:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:21:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:21:29.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:21:29.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:21:29.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:21:29.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:29.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:21:29.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:21:29.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:21:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:21:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:21:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:21:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:21:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:21:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:21:29.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:21:29.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:21:29.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:21:29.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:30.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-20T21:21:30.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:21:30.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:21:30.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:30.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:30.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:21:30.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:30.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:30.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:21:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:21:30.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:21:30.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:21:30.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:21:30.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:30.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-20T21:21:30.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:21:30.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:21:30.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:2 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:2 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:2 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:2 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:21:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:21:30.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:21:30.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:21:30.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-20T21:21:30.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:21:30.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:21:30.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:21:30.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:30.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:21:30.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:21:30.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:21:30.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:21:30.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:21:30.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:30.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:21:32.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test_resync_image started 2026-02-20T21:21:32.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:32.142 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:32.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:32.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:21:32.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:21:32.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:21:32.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:32.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:21:32.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:21:32.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:32.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:21:32.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:21:32.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:21:32.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:32.143 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:21:32.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:21:32.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:21:32.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:21:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:21:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:21:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:21:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:21:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:21:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:21:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:21:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:21:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:21:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:21:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:21:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:21:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:21:32.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:32.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:32.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:32.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:21:32.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:21:32.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:21:32.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:21:32.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:21:32.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:21:32.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:21:32.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:21:32.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:21:32.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:21:32.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:32.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-20T21:21:32.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:21:32.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:21:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:21:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:21:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:21:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:21:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:21:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:21:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:32.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-20T21:21:32.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:21:32.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:21:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:2 2026-02-20T21:21:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-20T21:21:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:2 2026-02-20T21:21:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:21:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:21:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-20T21:21:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:21:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:21:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:21:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-20T21:21:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:2 2026-02-20T21:21:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:21:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:32.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-20T21:21:32.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:2 2026-02-20T21:21:32.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:32.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:32.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:32.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:21:32.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-20T21:21:32.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:21:32.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:21:32.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:21:32.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:21:32.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-20T21:21:32.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:21:32.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:21:32.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:21:32.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:32.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:21:32.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:21:32.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:21:32.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:21:32.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:21:32.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:21:32.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:21:32.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:32.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+stopped 2026-02-20T21:21:32.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:32.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:32.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:32.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:21:32.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:21:32.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:32.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:32.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:33.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+stopped '' '' 2026-02-20T21:21:33.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:33.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:33.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:33.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:21:33.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:21:33.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:33.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:33.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:33.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-20T21:21:33.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_resync_image: 2026-02-20T21:21:33.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 9b0ec88c-a9ab-46dd-9407-72e81a713c20 2026-02-20T21:21:33.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:21:33.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:21:33.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:21:33.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:33 2026-02-20T21:21:33.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:21:33.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:21:33.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:21:33.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622488,"replay_state":"idle"} 2026-02-20T21:21:33.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:31 2026-02-20T21:21:33.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:21:33.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 189 .mirror.primary.9b0ec88c-a9ab-46dd-9407-72e81a713c20.5ad18a58-525d-43ad-ae5c-3e93ad2abac3 (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:21:33.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:21:33.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:21:33.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:21:33.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:21:33.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:21:33.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:21:33.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:21:33.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:21:33.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:21:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:21:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:21:33.353 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:21:33.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:33' 2026-02-20T21:21:33.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+stopped '2026-02-20 21:21:33' 2026-02-20T21:21:33.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:33.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:33.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:33.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:21:33.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:21:33' 2026-02-20T21:21:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:21:33.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK256715675mirror.0trial19920.2.0-677-gf7870454trueOK56775682mirror.1trial19920.2.0-677-gf7870454falseOK56845690mirror.2trial19920.2.0-677-gf7870454falseOK56925697mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}56715675mirror.0trial1992026-02-20 21:21:33cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:21:31test_resync_image9b0ec88c-a9ab-46dd-9407-72e81a713c20up+stoppedlocal image is primary56775682mirror.1trial1992026-02-20 21:21:33cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622488,"replay_state":"idle"}2026-02-20 21:21:31' 2026-02-20T21:21:33.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:21:33.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-20T21:21:33.389 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:33' 2026-02-20T21:21:33.389 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-20T21:21:33.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:21:33.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:21:33.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:21:33.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:21:33.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:33.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:33.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+replaying 2026-02-20T21:21:33.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:33.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:33.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:33.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:21:33.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:21:33.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:33.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:33.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:34.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+replaying '' '' 2026-02-20T21:21:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:21:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:21:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-02-20T21:21:34.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_resync_image: 2026-02-20T21:21:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 9b0ec88c-a9ab-46dd-9407-72e81a713c20 2026-02-20T21:21:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:21:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622488,"replay_state":"idle"} 2026-02-20T21:21:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.2 on trial199 2026-02-20T21:21:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:31 2026-02-20T21:21:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:21:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:21:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:21:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:21:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:33' 2026-02-20T21:21:34.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:21:34.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:21:34.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:21:34.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622488,"replay_state":"idle"} 2026-02-20T21:21:34.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:21:34.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:21:34.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:21:34.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:21:34.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:21:34.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.2 on trial199 2026-02-20T21:21:34.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:21:34.428 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:21:34.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:31' 2026-02-20T21:21:34.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+replaying '2026-02-20 21:21:31' 2026-02-20T21:21:34.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:34.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:34.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:34.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:21:34.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:21:31' 2026-02-20T21:21:34.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:34.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:34.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:21:34.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK245294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:21:31cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}2026-02-20 21:21:33test_resync_image9b0ec88c-a9ab-46dd-9407-72e81a713c20up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622488,"replay_state":"idle"}45364539mirror.2trial1992026-02-20 21:21:31cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:21:33' 2026-02-20T21:21:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:21:34.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-20T21:21:34.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:31' 2026-02-20T21:21:34.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-20T21:21:34.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:21:34.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:21:34.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:21:34.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:21:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror mirror test_resync_image 2026-02-20T21:21:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:21:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:21:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:21:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:21:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test_resync_image.export 2026-02-20T21:21:34.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test_resync_image.export 2026-02-20T21:21:34.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:21:34.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_resync_image.export 2026-02-20T21:21:34.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test_resync_image.export 2026-02-20T21:21:34.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test_resync_image.export 2026-02-20T21:21:34.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:21:34.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.export 2026-02-20T21:21:34.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_resync_image.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.export 2026-02-20T21:21:34.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/test_resync_image /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_resync_image.export 2026-02-20T21:21:34.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:21:34.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/test_resync_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.export 2026-02-20T21:21:34.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:21:34.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_resync_image.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.export 2026-02-20T21:21:34.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_resync_image.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.export 2026-02-20T21:21:34.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:34.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster1 mirror test_resync_image 2026-02-20T21:21:34.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:34.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:34.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:34.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:21:34.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:21:34.737 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster1 mirror test_resync_image 2026-02-20T21:21:34.737 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:34.737 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:34.737 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:34.738 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_resync_image 2026-02-20T21:21:34.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 rm mirror/test_resync_image 2026-02-20T21:21:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:21:35.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:35.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -z '' ']' 2026-02-20T21:21:35.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: request image resync when remote is not primary and daemon is offline' 2026-02-20T21:21:35.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:21:35.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:21:35.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:21:35 TEST: request image resync when remote is not primary and daemon is offline 2026-02-20T21:21:35.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:21:35 TEST: request image resync when remote is not primary and daemon is offline 2026-02-20T21:21:35.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_resync_image=test_resync_image 2026-02-20T21:21:35.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image_and_enable_mirror cluster2 mirror test_resync_image snapshot 2026-02-20T21:21:35.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:35.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:21:35.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:35.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:21:35.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:21:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:21:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:21:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:21:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image cluster2 mirror test_resync_image 2026-02-20T21:21:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:21:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:21:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:21:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=128 2026-02-20T21:21:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:21:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,object-map,fast-diff mirror/test_resync_image 2026-02-20T21:21:36.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:21:36.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:36.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:21:36.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:36.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror test_resync_image snapshot 2026-02-20T21:21:36.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:36.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:36.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:36.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:21:36.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_resync_image snapshot 2026-02-20T21:21:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:21:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/test_resync_image 2026-02-20T21:21:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test_resync_image': 2026-02-20T21:21:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:21:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:21:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:21:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 17908175d5b6 2026-02-20T21:21:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.17908175d5b6 2026-02-20T21:21:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:21:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:21:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:21:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:21:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:21:36 2026 2026-02-20T21:21:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:21:36 2026 2026-02-20T21:21:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:21:36 2026 2026-02-20T21:21:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:21:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:21:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 499f4fc0-0284-49c9-9cba-036b72f965b0 2026-02-20T21:21:36.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:21:36.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test_resync_image 100 2026-02-20T21:21:36.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:36.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:36.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:36.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:21:36.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:21:36.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:21:36.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:21:36.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test_resync_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:21:36.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:21:36.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:21:36.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5263.24 bytes/sec: 21 MiB/s 2026-02-20T21:21:36.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_resync_image 2026-02-20T21:21:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test_resync_image stopped 2026-02-20T21:21:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:21:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:21:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:37.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test_resync_image stopped 2026-02-20T21:21:37.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:37.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:37.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:37.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:21:37.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:21:37.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:21:37.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:37.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:21:37.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:21:37.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:37.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:21:37.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:21:37.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:21:37.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:37.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:21:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:21:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:21:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:21:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:21:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:21:37.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:37.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:37.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:21:37.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:21:37.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:21:37.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:21:37.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:37.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:21:37.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:21:37.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:21:37.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:21:37.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:21:37.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:37.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:21:37.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:21:37.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:37.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:37.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:37.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:21:37.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:21:37.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:21:37.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:21:37.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:21:37.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:21:37.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:21:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:21:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:21:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:21:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:37.514 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-20T21:21:37.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:21:37.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:21:37.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:37.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:37.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:21:37.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:37.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:37.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:21:37.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:21:37.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:37.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:37.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:21:37.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:21:37.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:21:37.519 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:21:37.520 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:37.520 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:21:37.520 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:21:37.520 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:21:37.520 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:21:37.520 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:21:37.520 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:37.520 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:21:37.520 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:21:37.520 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:37.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:37.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:37.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:21:37.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:21:37.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:21:37.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:21:37.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:21:37.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:21:37.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:21:37.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:21:37.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:21:37.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:21:37.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-20T21:21:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:21:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:21:37.576 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:37.576 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:37.576 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:2 2026-02-20T21:21:37.576 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:37.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:37.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-20T21:21:37.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:2 2026-02-20T21:21:37.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:37.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:37.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:21:37.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:21:37.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-20T21:21:37.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:21:37.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:37.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:21:37.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:21:37.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-20T21:21:37.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:2 2026-02-20T21:21:37.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:21:37.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:37.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-20T21:21:37.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:2 2026-02-20T21:21:37.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:37.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:37.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:37.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:21:37.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-20T21:21:37.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:21:37.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:21:37.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:21:37.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:21:37.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-20T21:21:37.584 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:21:37.584 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:21:37.584 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:21:37.584 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-20T21:21:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:21:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:21:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:3 2026-02-20T21:21:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-20T21:21:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:3 2026-02-20T21:21:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:21:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:21:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-20T21:21:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:21:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:21:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:21:37.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-02-20T21:21:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:3 2026-02-20T21:21:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:21:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-20T21:21:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:3 2026-02-20T21:21:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:21:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-20T21:21:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:21:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:21:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:21:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:21:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-20T21:21:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:21:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:21:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:21:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:37.680 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-20T21:21:37.680 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:21:37.680 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:21:37.681 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:37.681 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 1 2026-02-20T21:21:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:21:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:21:38.686 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:21:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:21:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:21:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:21:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:21:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:21:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:21:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:21:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:21:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:21:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:21:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:21:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:21:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:21:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:21:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:21:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:21:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:21:38.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:21:38.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:21:38.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:21:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:21:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:21:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:21:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-20T21:21:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:21:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:21:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:21:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:21:38.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:21:38.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:38.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:38.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:21:38.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:21:38.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:21:38.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:21:38.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:38.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:21:38.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:21:38.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:21:38.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:21:38.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:21:38.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:38.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:21:38.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:21:38.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:38.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:38.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:38.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:21:38.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:21:38.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:21:38.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:21:38.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:21:38.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:21:38.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:21:38.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:21:38.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:21:38.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:21:38.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:38.796 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-20T21:21:38.796 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:21:38.796 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:21:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:2 2026-02-20T21:21:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-20T21:21:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:2 2026-02-20T21:21:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:38.802 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:21:38.802 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:21:38.802 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-20T21:21:38.802 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:21:38.802 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:38.802 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:21:38.802 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:21:38.802 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-20T21:21:38.802 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:2 2026-02-20T21:21:38.802 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:21:38.802 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:38.802 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-20T21:21:38.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:2 2026-02-20T21:21:38.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:38.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:38.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:21:38.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:21:38.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-20T21:21:38.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:21:38.803 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:21:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:21:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:21:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-20T21:21:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:21:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:21:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:21:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:38.856 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:21:38.857 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:21:38.857 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:21:38.857 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:21:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:21:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test_resync_image 2026-02-20T21:21:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test_resync_image started 2026-02-20T21:21:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:38.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:21:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:21:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test_resync_image started 2026-02-20T21:21:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:21:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:21:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:21:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:21:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:21:39.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:21:39.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:21:39.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:39.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:21:39.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:21:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:21:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:21:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:21:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:21:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:21:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:21:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:21:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:21:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:21:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:21:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:21:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:21:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:21:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:21:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:21:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:21:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:21:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:21:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:21:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:21:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:21:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:21:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:21:39.864 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:21:39.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:21:39.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:21:39.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:21:39.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:39.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-20T21:21:39.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:21:39.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:21:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_resync_image 2026-02-20T21:21:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:21:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:21:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:21:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:21:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:21:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:21:39.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:21:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:21:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:21:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:21:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:21:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:21:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:21:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:21:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:21:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:21:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:21:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:21:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:21:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:21:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:21:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:21:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:21:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:21:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:21:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:21:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:21:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:21:39.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:21:39.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:21:39.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:21:39.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:21:39.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:21:39.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:21:39.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:21:39.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:39.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test_resync_image 2026-02-20T21:21:39.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:21:39.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:39.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:21:39.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:21:39.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:39.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:21:39.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:21:39.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test_resync_image 2026-02-20T21:21:39.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:39.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:39.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:39.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_resync_image 2026-02-20T21:21:40.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 186 2026-02-20T21:21:40.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test_resync_image 2026-02-20T21:21:40.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:21:40.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:40.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:21:40.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:21:40.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:40.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test_resync_image.status 2026-02-20T21:21:40.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test_resync_image.status 2026-02-20T21:21:40.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:21:40.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_resync_image.status 2026-02-20T21:21:40.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test_resync_image.status 2026-02-20T21:21:40.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test_resync_image.status 2026-02-20T21:21:40.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:21:40.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.status 2026-02-20T21:21:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_resync_image /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_resync_image.status 2026-02-20T21:21:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_resync_image.status 2026-02-20T21:21:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_resync_image --format xml 2026-02-20T21:21:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:21:41.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:21:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=186 2026-02-20T21:21:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:21:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:21:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:21:41.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_resync_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.status 2026-02-20T21:21:41.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:41.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:41.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:41.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.status 2026-02-20T21:21:41.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_resync_image --format xml 2026-02-20T21:21:41.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:21:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:21:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=185 2026-02-20T21:21:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 186 = 185 2026-02-20T21:21:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:21:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.4 2026-02-20T21:21:41.640 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_resync_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.status 2026-02-20T21:21:41.640 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:41.640 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:41.640 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:41.641 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.status 2026-02-20T21:21:41.641 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_resync_image --format xml 2026-02-20T21:21:41.641 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:21:41.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:21:41.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=185 2026-02-20T21:21:41.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 186 = 185 2026-02-20T21:21:41.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:21:41.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.8 2026-02-20T21:21:42.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_resync_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.status 2026-02-20T21:21:42.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:42.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:42.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:42.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.status 2026-02-20T21:21:42.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_resync_image --format xml 2026-02-20T21:21:42.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:21:42.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:21:42.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=186 2026-02-20T21:21:42.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 186 = 186 2026-02-20T21:21:42.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:42.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_resync_image 2026-02-20T21:21:42.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:42.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:42.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:42.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:21:42.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+replaying 2026-02-20T21:21:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:21:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:21:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+replaying '' '' 2026-02-20T21:21:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:21:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:21:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-20T21:21:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_resync_image: 2026-02-20T21:21:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 499f4fc0-0284-49c9-9cba-036b72f965b0 2026-02-20T21:21:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:21:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622496,"replay_state":"idle"} 2026-02-20T21:21:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:21:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:37 2026-02-20T21:21:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:21:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:21:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:21:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:21:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:41' 2026-02-20T21:21:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:21:43.543 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:21:43.543 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:21:43.543 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622496,"replay_state":"idle"} 2026-02-20T21:21:43.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:21:43.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:21:43.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:21:43.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:21:43.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:21:43.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:21:43.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:21:43.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:21:43.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:37' 2026-02-20T21:21:43.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+replaying '2026-02-20 21:21:37' 2026-02-20T21:21:43.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:43.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:43.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:43.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:21:43.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:21:37' 2026-02-20T21:21:43.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:43.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:43.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:21:43.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK256715675mirror.0trial19920.2.0-677-gf7870454trueOK56775682mirror.1trial19920.2.0-677-gf7870454falseOK56845690mirror.2trial19920.2.0-677-gf7870454falseOK56925697mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}56715675mirror.0trial1992026-02-20 21:21:33cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:21:41test_resync_image499f4fc0-0284-49c9-9cba-036b72f965b0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622496,"replay_state":"idle"}56775682mirror.1trial1992026-02-20 21:21:37cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:21:41' 2026-02-20T21:21:43.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:21:43.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-20T21:21:43.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:37' 2026-02-20T21:21:43.584 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-20T21:21:43.585 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:21:43.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:21:43.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:21:43.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:21:43.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:43.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:43.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+stopped 2026-02-20T21:21:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:21:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:21:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:44.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+stopped '' '' 2026-02-20T21:21:44.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:44.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:44.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:44.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:21:44.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:21:44.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:44.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:44.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:44.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-02-20T21:21:44.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_resync_image: 2026-02-20T21:21:44.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 499f4fc0-0284-49c9-9cba-036b72f965b0 2026-02-20T21:21:44.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:21:44.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:21:44.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.2 on trial199 2026-02-20T21:21:44.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:41 2026-02-20T21:21:44.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:21:44.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:21:44.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:21:44.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622496,"replay_state":"idle"} 2026-02-20T21:21:44.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:37 2026-02-20T21:21:44.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:21:44.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 185 .mirror.primary.499f4fc0-0284-49c9-9cba-036b72f965b0.dfd197c6-0d43-46c8-a64d-ee6ce0ea19bb (peer_uuids:[]) 2026-02-20T21:21:44.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 186 .mirror.primary.499f4fc0-0284-49c9-9cba-036b72f965b0.fbb43d0e-ec26-4bbf-a8e5-93168c4e2835 (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:21:44.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:21:44.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:21:44.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:21:44.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:21:44.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:21:44.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:21:44.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:21:44.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:21:44.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:21:44.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.2 on trial199 2026-02-20T21:21:44.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:21:44.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:21:44.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:41' 2026-02-20T21:21:44.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+stopped '2026-02-20 21:21:41' 2026-02-20T21:21:44.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:44.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:44.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:44.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:21:44.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:21:41' 2026-02-20T21:21:44.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:44.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:44.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:21:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK245294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:21:41cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}2026-02-20 21:21:33test_resync_image499f4fc0-0284-49c9-9cba-036b72f965b0up+stoppedlocal image is primary45364539mirror.2trial1992026-02-20 21:21:41cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622496,"replay_state":"idle"}2026-02-20 21:21:37' 2026-02-20T21:21:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:21:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-20T21:21:44.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:41' 2026-02-20T21:21:44.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-20T21:21:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:21:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:21:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:21:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:21:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirrors cluster1 2026-02-20T21:21:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:21:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:21:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:0 '' 2026-02-20T21:21:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:21:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:21:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:21:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:21:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:0 2026-02-20T21:21:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:0 2026-02-20T21:21:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:21:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:21:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:21:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:21:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:21:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:21:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:21:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:21:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:21:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:21:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-20T21:21:44.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:21:44.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:21:44.676 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=77763 2026-02-20T21:21:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 77763 ']' 2026-02-20T21:21:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 77763 2026-02-20T21:21:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:21:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:21:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:21:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=77763 '$2 == pid {print; exit 1}' 2026-02-20T21:21:44.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:ubuntu 77763 0.2 0.0 540592 52288 ? Ssl 21:21 0:00 rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:21:44.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:21:44.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:45.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:21:45.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=77763 '$2 == pid {print; exit 1}' 2026-02-20T21:21:45.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:21:45.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:21:45.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=77763 '$2 == pid {print; exit 1}' 2026-02-20T21:21:45.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:21:45.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:21:45.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:21:45.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:45.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:21:45.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:21:45.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:21:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:21:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:21:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:21:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:21:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:21:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:21:45.718 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:21:45.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:21:45.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:21:45.730 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-20T21:21:45.730 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:21:45.730 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:21:45.730 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:45.730 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:21:45.730 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:21:45.730 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:21:45.730 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:45.731 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:45.731 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:21:45.731 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:21:45.731 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:21:45.731 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:21:45.731 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:21:45.731 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:21:45.731 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:21:45.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:21:45.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:21:45.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:0 2026-02-20T21:21:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:21:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:21:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:21:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:21:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:21:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:21:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:21:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:21:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:21:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-20T21:21:45.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:21:45.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:21:45.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:45.756 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:1 '' 2026-02-20T21:21:45.756 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:21:45.756 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:21:45.756 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:21:45.756 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:21:45.756 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:1 2026-02-20T21:21:45.756 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:1 2026-02-20T21:21:45.757 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:45.757 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:21:45.757 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:21:45.757 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:21:45.757 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:45.757 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:21:45.757 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:21:45.757 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:21:45.757 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:21:45.757 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:21:45.757 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:21:45.757 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:21:45.758 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-20T21:21:45.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:21:45.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:21:45.769 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=77807 2026-02-20T21:21:45.769 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 77807 ']' 2026-02-20T21:21:45.769 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 77807 2026-02-20T21:21:45.769 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:21:45.769 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:21:45.770 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:21:45.770 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=77807 '$2 == pid {print; exit 1}' 2026-02-20T21:21:45.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:ubuntu 77807 0.5 0.0 0 0 ? Rs 21:21 0:00 [rbd-mirror] 2026-02-20T21:21:45.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:21:45.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:46.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:21:46.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=77807 '$2 == pid {print; exit 1}' 2026-02-20T21:21:46.794 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:21:46.794 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:21:46.794 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=77807 '$2 == pid {print; exit 1}' 2026-02-20T21:21:46.806 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:21:46.806 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:21:46.807 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:21:46.807 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:46.807 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:21:46.807 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:21:46.807 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:21:46.807 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:46.807 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:46.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:21:46.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:21:46.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:21:46.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:21:46.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:21:46.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:21:46.809 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:21:46.818 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:21:46.818 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:21:46.819 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-20T21:21:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:21:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:21:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:21:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:21:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:21:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:21:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:21:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:21:46.821 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:21:46.821 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:21:46.821 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:21:46.821 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:21:46.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:21:46.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:21:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:1 2026-02-20T21:21:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:21:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:21:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:21:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:21:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:21:46.835 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:21:46.835 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:21:46.835 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:21:46.835 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:21:46.835 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-20T21:21:46.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:21:46.843 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:21:46.845 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:46.845 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:2 '' 2026-02-20T21:21:46.845 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:21:46.845 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:21:46.845 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:21:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:21:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:2 2026-02-20T21:21:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:2 2026-02-20T21:21:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:21:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:21:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:21:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:21:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:21:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:21:46.847 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:21:46.847 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:21:46.847 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:21:46.847 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:21:46.847 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-20T21:21:46.856 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:21:46.856 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:21:46.857 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=77862 2026-02-20T21:21:46.857 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 77862 ']' 2026-02-20T21:21:46.857 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 77862 2026-02-20T21:21:46.858 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:21:46.858 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:21:46.858 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:21:46.858 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=77862 '$2 == pid {print; exit 1}' 2026-02-20T21:21:46.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:ubuntu 77862 0.2 0.0 469892 45332 ? Ssl 21:21 0:00 rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:21:46.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:21:46.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:47.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:21:47.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=77862 '$2 == pid {print; exit 1}' 2026-02-20T21:21:47.882 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:21:47.882 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:21:47.882 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=77862 '$2 == pid {print; exit 1}' 2026-02-20T21:21:47.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:21:47.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:21:47.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:21:47.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:47.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:21:47.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:21:47.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:21:47.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:47.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:47.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:21:47.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:21:47.894 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:21:47.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:21:47.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:21:47.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:21:47.895 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:21:47.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:21:47.906 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:21:47.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-20T21:21:47.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:21:47.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:21:47.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:47.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:21:47.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:21:47.909 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:21:47.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:47.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:47.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:21:47.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:21:47.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:21:47.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:21:47.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:21:47.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:21:47.911 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:21:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:21:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:21:47.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:2 2026-02-20T21:21:47.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:21:47.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:47.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:21:47.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:21:47.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:47.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:47.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:47.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:21:47.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:21:47.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:21:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:21:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:21:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:21:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-20T21:21:47.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:21:47.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:21:47.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:3 '' 2026-02-20T21:21:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:21:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:21:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:21:47.936 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:21:47.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:3 2026-02-20T21:21:47.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:3 2026-02-20T21:21:47.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:21:47.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:21:47.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:21:47.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:21:47.938 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:21:47.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:21:47.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:21:47.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:21:47.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:21:47.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:21:47.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:21:47.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:21:47.939 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-20T21:21:47.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:21:47.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:21:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=77941 2026-02-20T21:21:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 77941 ']' 2026-02-20T21:21:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 77941 2026-02-20T21:21:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:21:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:21:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:21:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=77941 '$2 == pid {print; exit 1}' 2026-02-20T21:21:47.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:ubuntu 77941 0.2 0.0 556976 46780 ? Rsl 21:21 0:00 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:21:47.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:21:47.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:21:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=77941 '$2 == pid {print; exit 1}' 2026-02-20T21:21:48.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:21:48.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:21:48.973 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=77941 '$2 == pid {print; exit 1}' 2026-02-20T21:21:48.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:21:48.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:21:48.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:21:48.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:48.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:21:48.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:21:48.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:21:48.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:48.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:48.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:21:48.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:21:48.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:21:48.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:21:48.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:21:48.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:21:48.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:21:48.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:21:48.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:21:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-20T21:21:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:21:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:21:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:21:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:21:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:21:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:21:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:21:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:21:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:21:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:21:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:21:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:21:49.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:21:49.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:21:49.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:3 2026-02-20T21:21:49.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:21:49.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:21:49.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:21:49.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:21:49.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:21:49.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:21:49.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:49.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:21:49.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:21:49.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:21:49.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:21:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:21:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:21:49.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-20T21:21:49.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:21:49.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:21:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test_resync_image 100 2026-02-20T21:21:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:21:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:21:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:21:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:21:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test_resync_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:21:49.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:21:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:21:49.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5263.24 bytes/sec: 21 MiB/s 2026-02-20T21:21:49.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test_resync_image 2026-02-20T21:21:49.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:49.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:49.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:49.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test_resync_image 2026-02-20T21:21:50.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:21:50.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ request_resync_image cluster1 mirror test_resync_image test_resync_image_id 2026-02-20T21:21:50.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:50.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:50.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:50.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id_var_name=test_resync_image_id 2026-02-20T21:21:50.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test_resync_image 2026-02-20T21:21:50.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:21:50.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:21:50.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test_resync_image 2026-02-20T21:21:50.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test_resync_image 2026-02-20T21:21:50.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:21:50.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval 'test_resync_image_id='\''16322758db28'\''' 2026-02-20T21:21:50.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test_resync_image_id=16322758db28 2026-02-20T21:21:50.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval 'test -n "$test_resync_image_id"' 2026-02-20T21:21:50.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -n 16322758db28 2026-02-20T21:21:50.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/test_resync_image 2026-02-20T21:21:50.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Flagged image for resync from primary 2026-02-20T21:21:50.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirrors cluster1 2026-02-20T21:21:50.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:50.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:21:50.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:50.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:0 2026-02-20T21:21:50.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:21:50.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:21:50.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:21:50.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:21:50.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:21:50.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:21:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:21:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:21:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:21:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:21:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:21:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:21:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:21:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:21:50.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:50.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:1 2026-02-20T21:21:50.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:21:50.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:21:50.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:21:50.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:21:50.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:21:50.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:21:50.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:21:50.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:21:50.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:21:50.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:21:50.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:50.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:21:50.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:21:50.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:21:50.352 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:21:50.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:50.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:2 2026-02-20T21:21:50.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:21:50.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:21:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:21:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:2 2026-02-20T21:21:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:21:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:21:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:21:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=2 2026-02-20T21:21:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:21:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:21:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=2 2026-02-20T21:21:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:21:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:21:50.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:21:50.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:21:50.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:3 2026-02-20T21:21:50.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:21:50.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:21:50.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:21:50.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:3 2026-02-20T21:21:50.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:21:50.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:21:50.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:21:50.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=3 2026-02-20T21:21:50.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:21:50.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:21:50.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:21:50.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=3 2026-02-20T21:21:50.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:21:50.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:21:50.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:21:50.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+unknown 'remote image is not primary' 2026-02-20T21:21:50.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:21:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-20T21:21:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:51.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-20T21:21:51.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:51.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:51.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:51.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:21:51.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-20T21:21:51.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:51.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:51.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:51.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-02-20T21:21:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_resync_image: 2026-02-20T21:21:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 499f4fc0-0284-49c9-9cba-036b72f965b0 2026-02-20T21:21:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:21:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:21:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.2 on trial199 2026-02-20T21:21:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:46 2026-02-20T21:21:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:21:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:21:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: down+unknown 2026-02-20T21:21:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: status not found 2026-02-20T21:21:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: ' 2026-02-20T21:21:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:21:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 1 2026-02-20T21:21:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:21:53.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-20T21:21:53.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:53.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:53.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:53.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:21:53.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-20T21:21:53.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:53.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:53.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:53.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-02-20T21:21:53.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_resync_image: 2026-02-20T21:21:53.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 499f4fc0-0284-49c9-9cba-036b72f965b0 2026-02-20T21:21:53.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:21:53.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:21:53.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.2 on trial199 2026-02-20T21:21:53.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:51 2026-02-20T21:21:53.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:21:53.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:21:53.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: down+unknown 2026-02-20T21:21:53.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: status not found 2026-02-20T21:21:53.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: ' 2026-02-20T21:21:53.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:21:53.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:21:53.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*remote image is not primary' 2026-02-20T21:21:53.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:21:53.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:21:53.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:21:53.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:21:53.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:21:53.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:21:53.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.2 on trial199 2026-02-20T21:21:53.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:21:53.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:21:53.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:51' 2026-02-20T21:21:53.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+unknown '2026-02-20 21:21:51' 2026-02-20T21:21:53.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:21:53.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:53.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:53.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:21:53.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:21:51' 2026-02-20T21:21:53.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:53.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:53.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:21:53.520 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING245294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:21:51cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebdown+unknownstatus not foundtest_resync_image499f4fc0-0284-49c9-9cba-036b72f965b0up+unknownremote image is not primary45364539mirror.2trial1992026-02-20 21:21:51cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebdown+unknownstatus not found' 2026-02-20T21:21:53.520 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:21:53.520 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-20T21:21:53.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:21:51' 2026-02-20T21:21:53.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-20T21:21:53.523 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:21:53.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:21:53.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:21:53.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:21:53.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:53.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:21:53.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' 2026-02-20T21:21:53.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:53.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:53.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:53.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:21:53.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-20T21:21:53.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:53.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:53.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:21:54.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-20T21:21:54.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:54.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:54.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:54.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:21:54.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-20T21:21:54.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:54.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:54.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:54.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-20T21:21:54.557 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_resync_image: 2026-02-20T21:21:54.557 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 499f4fc0-0284-49c9-9cba-036b72f965b0 2026-02-20T21:21:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:21:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622500,"remote_snapshot_timestamp":1771622509,"replay_state":"idle"} 2026-02-20T21:21:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:54 2026-02-20T21:21:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:21:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:21:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:21:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:21:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:51' 2026-02-20T21:21:54.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:21:54.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 1 2026-02-20T21:21:54.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:54.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:21:54.955 INFO:tasks.ceph.cluster1.osd.2.trial196.stderr:2026-02-20T21:21:54.953+0000 7f727e3ed640 -1 reset not still connected to 0x55abbb883110 2026-02-20T21:21:56.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-20T21:21:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:21:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:21:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:21:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:21:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-20T21:21:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:21:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:21:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:21:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-20T21:21:56.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_resync_image: 2026-02-20T21:21:56.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 499f4fc0-0284-49c9-9cba-036b72f965b0 2026-02-20T21:21:56.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:21:56.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image demoted 2026-02-20T21:21:56.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:21:56.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:55 2026-02-20T21:21:56.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:21:56.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:21:56.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:21:56.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:21:56.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:51' 2026-02-20T21:21:56.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:21:56.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:21:56.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*remote image is not primary' 2026-02-20T21:21:56.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 1 2026-02-20T21:21:56.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:21:56.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:22:00.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+unknown 'remote image is not primary' '' 2026-02-20T21:22:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:22:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:22:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'description_pattern=remote image is not primary' 2026-02-20T21:22:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:22:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:22:00.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:22:00.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-20T21:22:00.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_resync_image: 2026-02-20T21:22:00.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 499f4fc0-0284-49c9-9cba-036b72f965b0 2026-02-20T21:22:00.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:22:00.630 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:22:00.631 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:22:00.631 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:22:00 2026-02-20T21:22:00.631 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:22:00.631 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:22:00.631 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:22:00.631 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:22:00.631 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:21:56' 2026-02-20T21:22:00.631 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:22:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:22:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*remote image is not primary' 2026-02-20T21:22:00.633 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:22:00.633 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:22:00.633 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:22:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:22:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:22:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:22:00.635 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:22:00.635 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:22:00.635 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:22:00.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:22:00' 2026-02-20T21:22:00.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+unknown '2026-02-20 21:22:00' 2026-02-20T21:22:00.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:00.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:00.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:22:00.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:22:00.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:22:00' 2026-02-20T21:22:00.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:22:00.638 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:22:00.638 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:22:00.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING1158045810mirror.0trial19920.2.0-677-gf7870454trueOK58115813mirror.1trial19920.2.0-677-gf7870454falseOK58175824mirror.2trial19920.2.0-677-gf7870454falseOK58265832mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}58115813mirror.1trial1992026-02-20 21:21:54cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:21:56test_resync_image499f4fc0-0284-49c9-9cba-036b72f965b0up+unknownremote image is not primary58045810mirror.0trial1992026-02-20 21:22:00cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:21:56' 2026-02-20T21:22:00.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:22:00.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-20T21:22:00.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:22:00' 2026-02-20T21:22:00.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-20T21:22:00.672 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:22:00.672 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:22:00.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:22:00.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:22:00.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:00.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:00.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test_resync_image 2026-02-20T21:22:00.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:00.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:00.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:22:00.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:22:00.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_resync_image 2026-02-20T21:22:00.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:22:00.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test_resync_image 2026-02-20T21:22:00.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:00.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:00.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:22:00.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test_resync_image started 2026-02-20T21:22:00.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:00.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:00.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:22:00.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:22:00.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:22:00.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:22:00.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:22:01.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test_resync_image started 2026-02-20T21:22:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:22:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:22:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:22:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:22:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_resync_image 2026-02-20T21:22:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:22:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:22:01.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:22:01.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:22:01.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:22:01.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:22:01.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:22:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:22:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:22:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_resync_image 2026-02-20T21:22:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:22:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:22:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:22:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:22:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:22:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:22:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:22:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:22:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:22:01.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:22:01.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:22:01.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:22:01.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:22:01.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:01.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:22:01.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:22:01.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:22:01.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:01.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:22:01.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:22:01.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:22:01.970 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:22:01.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:22:01.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:22:01.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:22:01.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:22:01.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:22:01.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:22:01.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:22:01.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:22:02.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-20T21:22:02.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:22:02.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:22:02.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:02.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_resync_image 2026-02-20T21:22:02.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:22:02.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:02.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:02.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:22:02.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:22:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:22:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:22:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:22:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:22:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:22:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:22:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:22:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:22:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:22:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:22:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:02.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:22:02.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:22:02.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:22:02.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:02.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:22:02.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:22:02.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:22:02.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:22:02.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:22:02.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:22:02.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:22:02.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:22:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:22:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:22:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:22:02.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:22:02.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_resync_image'] 2026-02-20T21:22:02.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:22:02.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:22:02.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:02.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_resync_image 2026-02-20T21:22:02.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:2 2026-02-20T21:22:02.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:02.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:02.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-20T21:22:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:2 2026-02-20T21:22:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:22:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:22:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-20T21:22:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:22:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:22:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:22:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:22:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-20T21:22:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:2 2026-02-20T21:22:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:22:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-20T21:22:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:2 2026-02-20T21:22:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:22:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:22:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:22:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-20T21:22:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:22:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:22:02.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:22:02.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:22:02.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-20T21:22:02.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:22:02.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:22:02.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:22:02.089 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_resync_image 2026-02-20T21:22:02.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:22:02.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:22:02.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:22:02.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:22:02.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:22:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:22:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:22:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_resync_image up+stopped 2026-02-20T21:22:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:02.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:02.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:22:02.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:22:02.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:22:02.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:22:02.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:22:02.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:22:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+stopped '' '' 2026-02-20T21:22:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:03.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:03.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:22:03.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:22:03.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:22:03.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:22:03.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:22:03.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:22:03.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-20T21:22:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_resync_image: 2026-02-20T21:22:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 499f4fc0-0284-49c9-9cba-036b72f965b0 2026-02-20T21:22:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:22:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:22:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:22:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:22:00 2026-02-20T21:22:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:22:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:22:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:22:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622520,"replay_state":"idle"} 2026-02-20T21:22:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:22:01 2026-02-20T21:22:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:22:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 193 .mirror.primary.499f4fc0-0284-49c9-9cba-036b72f965b0.13a2076a-953c-4647-9f18-b8c07566a5ae (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:22:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:22:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 1 2026-02-20T21:22:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:22:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:22:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+stopped '' '' 2026-02-20T21:22:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:22:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:22:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:22:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:22:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:22:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:22:05.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-20T21:22:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_resync_image: 2026-02-20T21:22:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 499f4fc0-0284-49c9-9cba-036b72f965b0 2026-02-20T21:22:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:22:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:22:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:22:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:22:00 2026-02-20T21:22:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:22:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:22:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:22:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622520,"replay_state":"idle"} 2026-02-20T21:22:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:22:01 2026-02-20T21:22:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:22:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 193 .mirror.primary.499f4fc0-0284-49c9-9cba-036b72f965b0.13a2076a-953c-4647-9f18-b8c07566a5ae (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:22:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:22:05.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 1 2026-02-20T21:22:05.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:22:05.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:22:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_resync_image up+stopped '' '' 2026-02-20T21:22:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:22:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:22:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:22:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:22:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:22:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:22:09.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_resync_image 2026-02-20T21:22:09.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_resync_image: 2026-02-20T21:22:09.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 499f4fc0-0284-49c9-9cba-036b72f965b0 2026-02-20T21:22:09.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:22:09.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:22:09.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:22:09.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:22:05 2026-02-20T21:22:09.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:22:09.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:22:09.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:22:09.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622520,"replay_state":"idle"} 2026-02-20T21:22:09.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:22:01 2026-02-20T21:22:09.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:22:09.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 193 .mirror.primary.499f4fc0-0284-49c9-9cba-036b72f965b0.13a2076a-953c-4647-9f18-b8c07566a5ae (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:22:09.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:22:09.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:22:09.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:22:09.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:22:09.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:22:09.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:22:09.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:22:09.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:22:09.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:22:09.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:22:09.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:22:09.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:22:09.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:22:05' 2026-02-20T21:22:09.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_resync_image up+stopped '2026-02-20 21:22:05' 2026-02-20T21:22:09.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:22:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:22:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:22:05' 2026-02-20T21:22:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:22:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:22:09.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:22:09.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK258045810mirror.0trial19920.2.0-677-gf7870454trueOK58115813mirror.1trial19920.2.0-677-gf7870454falseOK58175824mirror.2trial19920.2.0-677-gf7870454falseOK58265832mirror.3trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}58115813mirror.1trial1992026-02-20 21:21:54cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:22:06test_resync_image499f4fc0-0284-49c9-9cba-036b72f965b0up+stoppedlocal image is primary58045810mirror.0trial1992026-02-20 21:22:05cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622520,"replay_state":"idle"}2026-02-20 21:22:01' 2026-02-20T21:22:09.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:22:09.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-20T21:22:09.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:22:05' 2026-02-20T21:22:09.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-20T21:22:09.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:22:09.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:22:09.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:22:09.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:22:09.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:09.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:09.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_resync_image up+replaying 2026-02-20T21:22:09.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:09.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:09.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:22:09.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:22:09.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:22:09.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:22:09.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:22:09.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:22:10.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test_resync_image up+replaying '' '' 2026-02-20T21:22:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:22:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:22:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:22:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:22:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:22:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:22:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_resync_image 2026-02-20T21:22:10.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_resync_image: 2026-02-20T21:22:10.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 499f4fc0-0284-49c9-9cba-036b72f965b0 2026-02-20T21:22:10.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:22:10.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622520,"replay_state":"idle"} 2026-02-20T21:22:10.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.2 on trial199 2026-02-20T21:22:10.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:22:01 2026-02-20T21:22:10.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:22:10.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:22:10.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:22:10.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:22:10.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:22:05' 2026-02-20T21:22:10.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:22:10.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:22:10.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:22:10.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622520,"replay_state":"idle"} 2026-02-20T21:22:10.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:22:10.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:22:10.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:22:10.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:22:10.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:22:10.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.2 on trial199 2026-02-20T21:22:10.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:22:10.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:22:10.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:22:01' 2026-02-20T21:22:10.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_resync_image up+replaying '2026-02-20 21:22:01' 2026-02-20T21:22:10.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:10.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:10.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:22:10.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:22:10.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:22:01' 2026-02-20T21:22:10.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:22:10.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:22:10.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:22:10.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK245294534mirror.1trial19920.2.0-677-gf7870454trueOK45364539mirror.2trial19920.2.0-677-gf7870454falseOK45434548mirror.3trial19920.2.0-677-gf7870454falseOK45584561mirror.0trial19920.2.0-677-gf7870454falseOKtest88d69193-4691-4616-9634-b744c1077c1eup+stoppedlocal image is primary45294534mirror.1trial1992026-02-20 21:22:06cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622305,"remote_snapshot_timestamp":1771622305,"replay_state":"idle"}2026-02-20 21:21:54test_resync_image499f4fc0-0284-49c9-9cba-036b72f965b0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622520,"replay_state":"idle"}45364539mirror.2trial1992026-02-20 21:22:01cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:22:05' 2026-02-20T21:22:10.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:22:10.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/last_update' 2026-02-20T21:22:10.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:22:01' 2026-02-20T21:22:10.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_resync_image'\'']/state' 2026-02-20T21:22:10.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:22:10.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:22:10.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:22:10.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:22:10.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:10.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:10.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror mirror test_resync_image 2026-02-20T21:22:10.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:22:10.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:22:10.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:10.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:22:10.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:22:10.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:22:10.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test_resync_image.export 2026-02-20T21:22:10.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test_resync_image.export 2026-02-20T21:22:10.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:22:10.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_resync_image.export 2026-02-20T21:22:10.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test_resync_image.export 2026-02-20T21:22:10.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test_resync_image.export 2026-02-20T21:22:10.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:22:10.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.export 2026-02-20T21:22:10.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_resync_image.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.export 2026-02-20T21:22:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/test_resync_image /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_resync_image.export 2026-02-20T21:22:10.448 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:22:10.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/test_resync_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.export 2026-02-20T21:22:10.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:22:10.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_resync_image.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.export 2026-02-20T21:22:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_resync_image.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_resync_image.export 2026-02-20T21:22:10.627 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:10.627 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster1 mirror test_resync_image 2026-02-20T21:22:10.627 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:10.627 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:10.627 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:22:10.627 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:22:10.627 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:22:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster1 mirror test_resync_image 2026-02-20T21:22:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_resync_image 2026-02-20T21:22:10.628 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_resync_image 2026-02-20T21:22:10.656 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 rm mirror/test_resync_image 2026-02-20T21:22:11.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:22:11.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:11.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: disable mirror while daemon is stopped' 2026-02-20T21:22:11.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:22:11.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:22:11.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:22:11 TEST: disable mirror while daemon is stopped 2026-02-20T21:22:11.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:22:11 TEST: disable mirror while daemon is stopped 2026-02-20T21:22:11.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirrors cluster1 2026-02-20T21:22:11.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:11.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:22:11.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:22:11.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:11.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:0 '' 2026-02-20T21:22:11.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:22:11.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:22:11.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:11.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:22:11.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:0 2026-02-20T21:22:11.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:0 2026-02-20T21:22:11.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:11.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:22:11.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:22:11.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:22:11.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:11.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:11.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:22:11.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:22:11.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:22:11.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:22:11.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:22:11.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:22:11.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-20T21:22:11.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:22:11.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:22:11.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=80015 2026-02-20T21:22:11.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 80015 ']' 2026-02-20T21:22:11.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 80015 2026-02-20T21:22:11.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:22:11.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:22:11.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:11.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=80015 '$2 == pid {print; exit 1}' 2026-02-20T21:22:12.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:ubuntu 80015 0.5 0.0 532396 50096 ? Ssl 21:21 0:00 rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:22:12.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:22:12.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:22:13.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:13.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=80015 '$2 == pid {print; exit 1}' 2026-02-20T21:22:13.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:22:13.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:13.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=80015 '$2 == pid {print; exit 1}' 2026-02-20T21:22:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:22:13.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:22:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:22:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:22:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:22:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:22:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:22:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:22:13.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:22:13.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:22:13.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:22:13.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:22:13.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:22:13.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:22:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-20T21:22:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:22:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:22:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:22:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:22:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:22:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:22:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:22:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:22:13.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:22:13.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:22:13.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:22:13.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:22:13.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:22:13.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:22:13.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:0 2026-02-20T21:22:13.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:22:13.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:13.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:22:13.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:22:13.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:13.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:13.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:13.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:22:13.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:22:13.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:22:13.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:13.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:22:13.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:22:13.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-20T21:22:13.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:22:13.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:22:13.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:1 '' 2026-02-20T21:22:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:22:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:22:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:22:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:1 2026-02-20T21:22:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:1 2026-02-20T21:22:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:22:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:22:13.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:22:13.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:13.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:13.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:22:13.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:22:13.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:22:13.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:22:13.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:22:13.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:22:13.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-20T21:22:13.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:22:13.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:22:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=80044 2026-02-20T21:22:13.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 80044 ']' 2026-02-20T21:22:13.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 80044 2026-02-20T21:22:13.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:22:13.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:22:13.086 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:13.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=80044 '$2 == pid {print; exit 1}' 2026-02-20T21:22:13.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:ubuntu 80044 0.4 0.0 475016 50572 ? Rsl 21:21 0:00 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:22:13.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:22:13.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:22:14.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:14.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=80044 '$2 == pid {print; exit 1}' 2026-02-20T21:22:14.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:22:14.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:14.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=80044 '$2 == pid {print; exit 1}' 2026-02-20T21:22:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:22:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:22:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:22:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:22:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:22:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:22:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:22:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:22:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:22:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:22:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:22:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:22:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:22:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:22:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-20T21:22:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:22:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:22:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:22:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:22:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:22:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:22:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:22:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:22:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:22:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:22:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:22:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:22:14.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:22:14.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:22:14.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:1 2026-02-20T21:22:14.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:22:14.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:14.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:22:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:22:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:22:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:22:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:22:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:22:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:22:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-20T21:22:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:22:14.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:22:14.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:14.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:2 '' 2026-02-20T21:22:14.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:22:14.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:22:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:22:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:2 2026-02-20T21:22:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:2 2026-02-20T21:22:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:22:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:22:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:22:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:22:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:22:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:22:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:22:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:22:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:22:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-20T21:22:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:22:14.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:22:14.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=80120 2026-02-20T21:22:14.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 80120 ']' 2026-02-20T21:22:14.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 80120 2026-02-20T21:22:14.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:22:14.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:22:14.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:14.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=80120 '$2 == pid {print; exit 1}' 2026-02-20T21:22:14.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:ubuntu 80120 0.3 0.0 466820 44008 ? Ssl 21:21 0:00 rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:22:14.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:22:14.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:22:15.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:15.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=80120 '$2 == pid {print; exit 1}' 2026-02-20T21:22:15.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:22:15.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:15.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=80120 '$2 == pid {print; exit 1}' 2026-02-20T21:22:15.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:22:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:22:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:22:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:22:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:22:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:22:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:22:15.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:22:15.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:22:15.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:22:15.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:22:15.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:22:15.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:22:15.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:22:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-20T21:22:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:22:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:22:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:22:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:22:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:22:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:22:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:22:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:22:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:22:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:22:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:22:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:22:15.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:22:15.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:22:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:2 2026-02-20T21:22:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:22:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:22:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:22:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:22:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:22:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:22:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:22:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:22:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-20T21:22:15.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:22:15.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:22:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:3 '' 2026-02-20T21:22:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:22:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:22:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:22:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:3 2026-02-20T21:22:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:3 2026-02-20T21:22:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:22:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:22:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:22:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:22:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:22:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:22:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:22:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:22:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:22:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-20T21:22:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:22:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:22:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=80198 2026-02-20T21:22:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 80198 ']' 2026-02-20T21:22:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 80198 2026-02-20T21:22:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:22:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:22:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=80198 '$2 == pid {print; exit 1}' 2026-02-20T21:22:15.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:ubuntu 80198 0.3 0.0 264988 45224 ? Ssl 21:21 0:00 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:22:15.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:22:15.282 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:22:16.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:16.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=80198 '$2 == pid {print; exit 1}' 2026-02-20T21:22:16.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:22:16.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:16.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=80198 '$2 == pid {print; exit 1}' 2026-02-20T21:22:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:22:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:22:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:22:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:22:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:22:16.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:22:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:22:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:22:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:22:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:22:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:22:16.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:22:16.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:22:16.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:22:16.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-20T21:22:16.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:22:16.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:22:16.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:16.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:22:16.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:22:16.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:22:16.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:16.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:16.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:22:16.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:22:16.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:22:16.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:22:16.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:22:16.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:22:16.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:22:16.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:22:16.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:22:16.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:3 2026-02-20T21:22:16.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:22:16.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:16.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:22:16.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:22:16.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:16.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:16.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:16.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:22:16.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:22:16.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:22:16.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:16.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:22:16.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:22:16.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-20T21:22:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:22:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:22:16.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirrors cluster2 2026-02-20T21:22:16.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:16.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:22:16.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:22:16.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:16.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster2:0 '' 2026-02-20T21:22:16.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2:0 2026-02-20T21:22:16.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:22:16.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:16.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:22:16.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster2:0 2026-02-20T21:22:16.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2:0 2026-02-20T21:22:16.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:16.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:22:16.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:22:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:22:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:22:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:22:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:22:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster2 2026-02-20T21:22:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster2 2026-02-20T21:22:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:22:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:22:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-02-20T21:22:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-20T21:22:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-20T21:22:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=54170 2026-02-20T21:22:16.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 54170 ']' 2026-02-20T21:22:16.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 54170 2026-02-20T21:22:16.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:22:16.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:22:16.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:16.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=54170 '$2 == pid {print; exit 1}' 2026-02-20T21:22:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:ubuntu 54170 0.0 0.0 196356 62760 ? Rsl 21:05 0:00 rbd-mirror --cluster cluster2 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:22:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:22:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:22:17.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:17.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=54170 '$2 == pid {print; exit 1}' 2026-02-20T21:22:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:22:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=54170 '$2 == pid {print; exit 1}' 2026-02-20T21:22:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster2:0 cluster1 2026-02-20T21:22:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster2:0 2026-02-20T21:22:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:22:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:22:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:22:17.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:17.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:22:17.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:22:17.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:22:17.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster2 2026-02-20T21:22:17.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster2 2026-02-20T21:22:17.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:22:17.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:22:17.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:22:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:22:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:22:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:22:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster2:0 2026-02-20T21:22:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:22:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:22:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:22:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:22:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:17.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:22:17.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:22:17.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:22:17.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster2 2026-02-20T21:22:17.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster2 2026-02-20T21:22:17.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:22:17.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:22:17.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:22:17.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:22:17.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:22:17.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster2:0 2026-02-20T21:22:17.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:22:17.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:17.423 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:22:17.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:22:17.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:17.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:17.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:22:17.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:22:17.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:22:17.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:22:17.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:22:17.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:22:17.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:22:17.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-02-20T21:22:17.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-20T21:22:17.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-20T21:22:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster2:1 '' 2026-02-20T21:22:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2:1 2026-02-20T21:22:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:22:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:22:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster2:1 2026-02-20T21:22:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2:1 2026-02-20T21:22:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:22:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:22:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:22:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:22:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:22:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:22:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster2 2026-02-20T21:22:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster2 2026-02-20T21:22:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:22:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:22:17.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'pid file' 2026-02-20T21:22:17.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-20T21:22:17.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-20T21:22:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=54190 2026-02-20T21:22:17.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 54190 ']' 2026-02-20T21:22:17.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 54190 2026-02-20T21:22:17.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:22:17.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:22:17.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:17.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=54190 '$2 == pid {print; exit 1}' 2026-02-20T21:22:17.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:ubuntu 54190 0.1 0.0 237336 64204 ? Rsl 21:05 0:01 rbd-mirror --cluster cluster2 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:22:17.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:22:17.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:22:18.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:18.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=54190 '$2 == pid {print; exit 1}' 2026-02-20T21:22:18.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:22:18.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:18.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=54190 '$2 == pid {print; exit 1}' 2026-02-20T21:22:18.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster2:1 cluster1 2026-02-20T21:22:18.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster2:1 2026-02-20T21:22:18.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:18.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:18.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:22:18.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:22:18.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:22:18.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:18.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:22:18.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:22:18.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:22:18.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster2 2026-02-20T21:22:18.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster2 2026-02-20T21:22:18.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:22:18.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:22:18.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:22:18.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:22:18.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:22:18.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:22:18.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster2:1 2026-02-20T21:22:18.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:22:18.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:18.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:22:18.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:22:18.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:22:18.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:18.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:22:18.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:22:18.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:22:18.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster2 2026-02-20T21:22:18.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster2 2026-02-20T21:22:18.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:22:18.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:22:18.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:22:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:22:18.514 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:22:18.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster2:1 2026-02-20T21:22:18.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:22:18.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:18.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:22:18.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:22:18.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:22:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:22:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:22:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:22:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:22:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:22:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:22:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'pid file' 2026-02-20T21:22:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-20T21:22:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-20T21:22:18.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:18.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster2:2 '' 2026-02-20T21:22:18.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2:2 2026-02-20T21:22:18.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:22:18.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:18.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:22:18.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster2:2 2026-02-20T21:22:18.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2:2 2026-02-20T21:22:18.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:18.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:2 cluster instance 2026-02-20T21:22:18.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:2 2026-02-20T21:22:18.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:22:18.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:18.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:22:18.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:22:18.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-20T21:22:18.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster2 2026-02-20T21:22:18.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster2 2026-02-20T21:22:18.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:22:18.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:22:18.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'pid file' 2026-02-20T21:22:18.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-20T21:22:18.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-20T21:22:18.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=54234 2026-02-20T21:22:18.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 54234 ']' 2026-02-20T21:22:18.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 54234 2026-02-20T21:22:18.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:22:18.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:22:18.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:18.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=54234 '$2 == pid {print; exit 1}' 2026-02-20T21:22:18.557 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:ubuntu 54234 0.1 0.0 384864 65788 ? Ssl 21:05 0:01 rbd-mirror --cluster cluster2 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:22:18.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:22:18.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:22:19.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:19.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=54234 '$2 == pid {print; exit 1}' 2026-02-20T21:22:19.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:22:19.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:19.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=54234 '$2 == pid {print; exit 1}' 2026-02-20T21:22:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster2:2 cluster1 2026-02-20T21:22:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster2:2 2026-02-20T21:22:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-20T21:22:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:2 2026-02-20T21:22:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:22:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:22:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:22:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-20T21:22:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster2 2026-02-20T21:22:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster2 2026-02-20T21:22:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:22:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:22:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-20T21:22:19.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:22:19.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:22:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-20T21:22:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster2:2 2026-02-20T21:22:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:22:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-20T21:22:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:2 2026-02-20T21:22:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:22:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:22:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:22:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-20T21:22:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster2 2026-02-20T21:22:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster2 2026-02-20T21:22:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:22:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:22:19.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-20T21:22:19.607 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:22:19.607 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:22:19.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster2:2 2026-02-20T21:22:19.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:2 2026-02-20T21:22:19.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:19.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-20T21:22:19.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:2 2026-02-20T21:22:19.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:19.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:19.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:22:19.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:22:19.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-20T21:22:19.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:22:19.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:22:19.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:22:19.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:22:19.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'pid file' 2026-02-20T21:22:19.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-20T21:22:19.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-20T21:22:19.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:19.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster2:3 '' 2026-02-20T21:22:19.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2:3 2026-02-20T21:22:19.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:22:19.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:19.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:22:19.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster2:3 2026-02-20T21:22:19.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2:3 2026-02-20T21:22:19.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:19.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:3 cluster instance 2026-02-20T21:22:19.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:3 2026-02-20T21:22:19.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:22:19.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:19.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:22:19.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:22:19.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-20T21:22:19.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster2 2026-02-20T21:22:19.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster2 2026-02-20T21:22:19.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:22:19.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:22:19.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'pid file' 2026-02-20T21:22:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-20T21:22:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-20T21:22:19.636 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=54295 2026-02-20T21:22:19.636 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 54295 ']' 2026-02-20T21:22:19.636 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 54295 2026-02-20T21:22:19.636 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:22:19.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:22:19.638 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:19.638 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=54295 '$2 == pid {print; exit 1}' 2026-02-20T21:22:19.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:ubuntu 54295 0.0 0.0 458624 59176 ? Rsl 21:05 0:00 rbd-mirror --cluster cluster2 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:22:19.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:22:19.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:22:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=54295 '$2 == pid {print; exit 1}' 2026-02-20T21:22:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:22:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:22:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=54295 '$2 == pid {print; exit 1}' 2026-02-20T21:22:20.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster2:3 cluster1 2026-02-20T21:22:20.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster2:3 2026-02-20T21:22:20.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:20.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:20.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-20T21:22:20.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:3 2026-02-20T21:22:20.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:22:20.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:20.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:22:20.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:22:20.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-20T21:22:20.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster2 2026-02-20T21:22:20.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster2 2026-02-20T21:22:20.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:22:20.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:22:20.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-20T21:22:20.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:22:20.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:22:20.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-20T21:22:20.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster2:3 2026-02-20T21:22:20.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:22:20.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:20.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-20T21:22:20.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:3 2026-02-20T21:22:20.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:22:20.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:20.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:22:20.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:22:20.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-20T21:22:20.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster2 2026-02-20T21:22:20.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster2 2026-02-20T21:22:20.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:22:20.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:22:20.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-20T21:22:20.697 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:22:20.697 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:22:20.699 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster2:3 2026-02-20T21:22:20.699 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:3 2026-02-20T21:22:20.700 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:20.700 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-20T21:22:20.700 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:3 2026-02-20T21:22:20.700 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:20.700 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:20.700 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:22:20.700 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:22:20.700 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-20T21:22:20.700 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:22:20.700 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:22:20.700 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:22:20.700 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:22:20.701 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'pid file' 2026-02-20T21:22:20.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-20T21:22:20.710 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-20T21:22:20.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:22:20.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ disable_mirror cluster2 mirror test 2026-02-20T21:22:20.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:20.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:20.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:20.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/test 2026-02-20T21:22:20.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring disabled 2026-02-20T21:22:20.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -z '' ']' 2026-02-20T21:22:20.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test present 2026-02-20T21:22:20.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=present 2026-02-20T21:22:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:22:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:22:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:22:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:22:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:22:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:22:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:22:20.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:22:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id=1139b6cec650 2026-02-20T21:22:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n 1139b6cec650 2026-02-20T21:22:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -z '' -o '' = 1139b6cec650 2026-02-20T21:22:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=present 2026-02-20T21:22:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test present = present 2026-02-20T21:22:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirrors cluster1 2026-02-20T21:22:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:21.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:22:21.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:21.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:0 2026-02-20T21:22:21.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:22:21.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:22:21.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:22:21.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:22:21.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:22:21.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:22:21.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:22:21.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:22:21.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:22:21.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:22:21.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:21.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:22:21.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:22:21.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:21.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:22:21.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:21.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:1 2026-02-20T21:22:21.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:22:21.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:22:21.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:22:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:22:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:22:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:22:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:22:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:22:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:22:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:22:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:22:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:22:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:21.040 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:22:21.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:2 2026-02-20T21:22:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:22:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:22:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:22:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:2 2026-02-20T21:22:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:22:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:22:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:22:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=2 2026-02-20T21:22:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:22:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:22:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:21.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=2 2026-02-20T21:22:21.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:22:21.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:21.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:22:21.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:21.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:3 2026-02-20T21:22:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:22:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:22:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:22:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:3 2026-02-20T21:22:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:22:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:22:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:22:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=3 2026-02-20T21:22:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:22:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:22:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=3 2026-02-20T21:22:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:22:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:22:21.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror test deleted 2026-02-20T21:22:21.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:21.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:21.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:21.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=deleted 2026-02-20T21:22:21.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:22:21.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:22:21.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:21.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:22:21.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:21.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:22:21.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:22:21.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:22:21.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:22:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image_id=1139b6cec650 2026-02-20T21:22:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:22:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:22:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test deleted 1139b6cec650 2026-02-20T21:22:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=deleted 2026-02-20T21:22:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id=1139b6cec650 2026-02-20T21:22:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:22:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:22:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:22:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:22:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:22:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:22:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:22:21.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id=1139b6cec650 2026-02-20T21:22:21.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n 1139b6cec650 2026-02-20T21:22:21.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -z 1139b6cec650 -o 1139b6cec650 = 1139b6cec650 2026-02-20T21:22:21.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=present 2026-02-20T21:22:21.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test deleted = present 2026-02-20T21:22:21.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:22:21.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:22:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test deleted 1139b6cec650 2026-02-20T21:22:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=deleted 2026-02-20T21:22:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id=1139b6cec650 2026-02-20T21:22:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:22:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:22:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:22:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:22:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:22:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:22:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:22:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id=1139b6cec650 2026-02-20T21:22:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n 1139b6cec650 2026-02-20T21:22:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -z 1139b6cec650 -o 1139b6cec650 = 1139b6cec650 2026-02-20T21:22:22.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=present 2026-02-20T21:22:22.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test deleted = present 2026-02-20T21:22:22.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:22:22.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:22:24.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test deleted 1139b6cec650 2026-02-20T21:22:24.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:24.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:24.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:24.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=deleted 2026-02-20T21:22:24.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id=1139b6cec650 2026-02-20T21:22:24.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:22:24.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:22:24.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:22:24.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:24.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:22:24.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:22:24.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:22:24.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:22:24.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id=1139b6cec650 2026-02-20T21:22:24.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n 1139b6cec650 2026-02-20T21:22:24.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -z 1139b6cec650 -o 1139b6cec650 = 1139b6cec650 2026-02-20T21:22:24.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=present 2026-02-20T21:22:24.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test deleted = present 2026-02-20T21:22:24.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:22:24.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:22:28.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test deleted 1139b6cec650 2026-02-20T21:22:28.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:28.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:28.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:28.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=deleted 2026-02-20T21:22:28.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id=1139b6cec650 2026-02-20T21:22:28.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:22:28.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:22:28.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:22:28.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:28.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:22:28.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:22:28.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:22:28.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:22:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:22:28.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id= 2026-02-20T21:22:28.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:28.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test deleted = deleted 2026-02-20T21:22:28.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:28.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:22:28.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror test 2026-02-20T21:22:28.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:28.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:28.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:28.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:22:28.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2026-02-20T21:22:29.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:22:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/test 2026-02-20T21:22:29.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test': 2026-02-20T21:22:29.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:22:29.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:22:29.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:22:29.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 119ca81aad17 2026-02-20T21:22:29.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.119ca81aad17 2026-02-20T21:22:29.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:22:29.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:22:29.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:22:29.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:22:29.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:05:45 2026 2026-02-20T21:22:29.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:07:00 2026 2026-02-20T21:22:29.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:07:09 2026 2026-02-20T21:22:29.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:22:29.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:22:29.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: d5138118-365e-4c80-9761-89b69f10c700 2026-02-20T21:22:29.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:22:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror test present 2026-02-20T21:22:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=present 2026-02-20T21:22:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:22:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:22:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:22:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:22:29.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:22:29.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:22:29.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:22:29.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:22:29.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image_id= 2026-02-20T21:22:29.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:22:29.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:22:29.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-20T21:22:29.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:29.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:29.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:29.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=present 2026-02-20T21:22:29.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:22:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:22:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:22:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:22:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:22:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:22:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:22:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:22:29.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:22:29.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id= 2026-02-20T21:22:29.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:29.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test present = deleted 2026-02-20T21:22:29.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:22:29.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:22:30.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-20T21:22:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=present 2026-02-20T21:22:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:22:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:22:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:22:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:22:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:22:30.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:22:30.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:22:30.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:22:30.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:22:30.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id= 2026-02-20T21:22:30.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:30.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test present = deleted 2026-02-20T21:22:30.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:22:30.375 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:22:32.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-20T21:22:32.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:32.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:32.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:32.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=present 2026-02-20T21:22:32.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:22:32.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:22:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:22:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:22:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:22:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:22:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:22:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:22:32.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id=16faa5eeddfc 2026-02-20T21:22:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n 16faa5eeddfc 2026-02-20T21:22:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -z '' -o '' = 16faa5eeddfc 2026-02-20T21:22:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=present 2026-02-20T21:22:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test present = present 2026-02-20T21:22:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:22:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:22:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:22:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:22:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:22:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:22:33.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:22:33.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:33.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:22:33.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:33.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:22:33.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:22:33.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:22:33.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:22:33.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:22:33.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:22:33.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:33.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:33.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:22:33.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:22:33.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:22:33.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:22:33.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:22:33.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:22:33.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:33.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:22:33.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:22:33.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:33.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:33.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:22:33.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:22:33.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:33.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:33.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:33.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:22:33.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:22:33.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:22:33.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:33.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:22:33.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:22:33.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:22:33.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:22:33.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:22:33.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:33.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:22:33.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:22:33.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:22:33.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:33.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:33.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:22:33.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:22:33.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:22:33.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:22:33.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:22:33.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:22:33.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:22:33.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:22:33.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:22:33.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:22:33.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:22:33.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:22:33.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:22:33.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:22:33.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:22:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:22:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:22:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:22:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:33.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:22:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:22:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:22:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:22:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:22:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:22:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:22:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:22:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:22:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:22:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:22:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:22:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:22:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:22:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:22:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:22:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:22:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:22:33.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:22:33.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:22:33.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:22:33.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:22:33.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:22:33.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:22:33.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:22:33.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:22:33.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:22:33.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:22:33.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:22:33.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:33.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: non-default namespace image mirroring' 2026-02-20T21:22:33.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:22:33.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:22:33.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:22:33 TEST: non-default namespace image mirroring 2026-02-20T21:22:33.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:22:33 TEST: non-default namespace image mirroring 2026-02-20T21:22:33.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog ' - replay' 2026-02-20T21:22:33.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:22:33.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:22:33.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:22:33 - replay 2026-02-20T21:22:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:22:33 - replay 2026-02-20T21:22:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image_and_enable_mirror cluster2 mirror/ns1 test snapshot 2026-02-20T21:22:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:22:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:22:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:22:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image cluster2 mirror/ns1 test 2026-02-20T21:22:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:22:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=128 2026-02-20T21:22:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:22:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,object-map,fast-diff mirror/ns1/test 2026-02-20T21:22:33.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror/ns1 2026-02-20T21:22:33.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:22:33.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:22:33.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:33.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror/ns1 test snapshot 2026-02-20T21:22:33.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:33.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:22:33.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:33.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:22:33.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/ns1/test snapshot 2026-02-20T21:22:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:22:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/ns1/test 2026-02-20T21:22:34.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test': 2026-02-20T21:22:34.004 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:22:34.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:22:34.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:22:34.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 17e9420be9fe 2026-02-20T21:22:34.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.17e9420be9fe 2026-02-20T21:22:34.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:22:34.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:22:34.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:22:34.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:22:34.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:22:33 2026 2026-02-20T21:22:34.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:22:33 2026 2026-02-20T21:22:34.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:22:33 2026 2026-02-20T21:22:34.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:22:34.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:22:34.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: cdfe5f4d-6ee5-4727-a7e9-c6361dbd5990 2026-02-20T21:22:34.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:22:34.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image_and_enable_mirror cluster2 mirror/ns2 test snapshot 2026-02-20T21:22:34.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:34.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:34.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns2 2026-02-20T21:22:34.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:34.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:34.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:34.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:22:34.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:22:34.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:34.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image cluster2 mirror/ns2 test 2026-02-20T21:22:34.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:34.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:34.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns2 2026-02-20T21:22:34.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:34.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:34.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:34.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=128 2026-02-20T21:22:34.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:22:34.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,object-map,fast-diff mirror/ns2/test 2026-02-20T21:22:34.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror/ns2 2026-02-20T21:22:34.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns2 2026-02-20T21:22:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:22:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror/ns2 test snapshot 2026-02-20T21:22:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns2 2026-02-20T21:22:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:22:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/ns2/test snapshot 2026-02-20T21:22:34.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:22:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/ns2/test 2026-02-20T21:22:35.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test': 2026-02-20T21:22:35.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:22:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:22:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:22:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 17f2af11df6 2026-02-20T21:22:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.17f2af11df6 2026-02-20T21:22:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:22:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:22:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:22:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:22:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:22:34 2026 2026-02-20T21:22:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:22:34 2026 2026-02-20T21:22:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:22:34 2026 2026-02-20T21:22:35.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:22:35.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:22:35.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 6c61272f-0aa4-4613-863f-a017aa5e6dd1 2026-02-20T21:22:35.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:22:35.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror/ns1 test 2026-02-20T21:22:35.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:35.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:22:35.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:35.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror/ns1 test started 2026-02-20T21:22:35.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:35.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:22:35.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:35.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:22:35.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:22:35.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:22:35.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:22:36.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror/ns1 test started 2026-02-20T21:22:36.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:36.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:22:36.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:36.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:22:36.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:22:36.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:22:36.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/ns1/test 2026-02-20T21:22:36.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:22:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:22:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:22:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:22:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:22:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:22:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:22:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:36.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:22:36.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:22:36.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:22:36.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:22:36.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:22:36.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:22:36.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:22:36.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:22:36.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:22:36.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:22:36.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test 2026-02-20T21:22:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-02-20T21:22:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:22:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:22:36.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:36.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns1/test 2026-02-20T21:22:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:22:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:22:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:22:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:22:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:22:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:22:36.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:36.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:22:36.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:22:36.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:22:36.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:22:36.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:22:36.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:36.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:22:36.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:22:36.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:22:36.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:36.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:22:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:22:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:22:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:22:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:22:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:22:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:22:36.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:22:36.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:22:36.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:22:36.084 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns1/test 2026-02-20T21:22:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:22:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:22:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:22:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:22:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:22:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:22:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:22:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror/ns2 test 2026-02-20T21:22:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns2 2026-02-20T21:22:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror/ns2 test started 2026-02-20T21:22:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns2 2026-02-20T21:22:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:22:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:22:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:22:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:22:37.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror/ns2 test started 2026-02-20T21:22:37.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:37.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns2 2026-02-20T21:22:37.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:37.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:22:37.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:22:37.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:22:37.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/ns2/test 2026-02-20T21:22:37.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:22:37.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:37.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:37.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:22:37.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:22:37.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:22:37.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:22:37.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:22:37.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:22:37.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:22:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns2/test 2026-02-20T21:22:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:22:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:22:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:22:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:22:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:22:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:22:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:22:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:22:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:22:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:22:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:22:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:37.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:22:37.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:22:37.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:22:37.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:37.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:37.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:22:37.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:22:37.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:22:37.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:22:37.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:22:37.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:22:37.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:22:37.148 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:22:37.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:22:37.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:22:37.149 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns2/test 2026-02-20T21:22:37.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns2/test'] 2026-02-20T21:22:37.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:22:37.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:22:37.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:37.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns2/test 2026-02-20T21:22:37.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:22:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:22:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:22:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:22:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:22:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:22:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:22:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:22:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:22:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:22:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:22:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:22:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:22:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:22:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:22:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:22:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:22:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:22:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:22:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:22:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:22:37.204 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:22:37.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:22:37.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:22:37.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns2/test 2026-02-20T21:22:37.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:22:37.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:22:37.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:22:37.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:22:37.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:22:37.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:22:37.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:22:37.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:37.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror/ns1 test 100 2026-02-20T21:22:37.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:37.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:22:37.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:37.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:22:37.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:22:37.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:37.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:22:37.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/ns1/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:22:37.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:22:37.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:22:37.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5263.24 bytes/sec: 21 MiB/s 2026-02-20T21:22:37.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror/ns2 test 100 2026-02-20T21:22:37.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns2 2026-02-20T21:22:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:22:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:22:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:22:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/ns2/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:22:37.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:22:37.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:22:37.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 4761.98 bytes/sec: 19 MiB/s 2026-02-20T21:22:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror/ns1 mirror/ns1 test 2026-02-20T21:22:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:22:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror/ns1 2026-02-20T21:22:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror/ns1 2026-02-20T21:22:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:22:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:22:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror/ns1 test 2026-02-20T21:22:37.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:37.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:22:37.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:37.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/ns1/test 2026-02-20T21:22:37.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 192 2026-02-20T21:22:37.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror/ns1 mirror/ns1 test 2026-02-20T21:22:37.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:22:37.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:37.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror/ns1 2026-02-20T21:22:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror/ns1 2026-02-20T21:22:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror/ns1-test.status 2026-02-20T21:22:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror/ns1-test.status 2026-02-20T21:22:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:22:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test.status 2026-02-20T21:22:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror/ns1-test.status 2026-02-20T21:22:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror/ns1-test.status 2026-02-20T21:22:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.status 2026-02-20T21:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror/ns1 test /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test.status 2026-02-20T21:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test.status 2026-02-20T21:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/ns1/test --format xml 2026-02-20T21:22:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:22:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:22:38.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=192 2026-02-20T21:22:38.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:22:38.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:22:38.020 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:22:38.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.status 2026-02-20T21:22:38.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:38.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:22:38.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:38.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.status 2026-02-20T21:22:38.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test --format xml 2026-02-20T21:22:38.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:22:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:22:38.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=190 2026-02-20T21:22:38.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 192 = 190 2026-02-20T21:22:38.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:22:38.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.4 2026-02-20T21:22:38.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.status 2026-02-20T21:22:38.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:38.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:22:38.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:38.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.status 2026-02-20T21:22:38.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test --format xml 2026-02-20T21:22:38.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:22:38.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:22:38.686 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=190 2026-02-20T21:22:38.686 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 192 = 190 2026-02-20T21:22:38.686 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:22:38.686 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.8 2026-02-20T21:22:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.status 2026-02-20T21:22:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:22:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.status 2026-02-20T21:22:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test --format xml 2026-02-20T21:22:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:22:39.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:22:39.523 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=192 2026-02-20T21:22:39.523 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 192 = 192 2026-02-20T21:22:39.523 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:39.523 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror/ns2 mirror/ns2 test 2026-02-20T21:22:39.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:22:39.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:39.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror/ns2 2026-02-20T21:22:39.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror/ns2 2026-02-20T21:22:39.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:39.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:22:39.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:22:39.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror/ns2 test 2026-02-20T21:22:39.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:39.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns2 2026-02-20T21:22:39.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:39.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/ns2/test 2026-02-20T21:22:39.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 193 2026-02-20T21:22:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror/ns2 mirror/ns2 test 2026-02-20T21:22:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:22:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror/ns2 2026-02-20T21:22:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror/ns2 2026-02-20T21:22:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror/ns2-test.status 2026-02-20T21:22:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror/ns2-test.status 2026-02-20T21:22:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:22:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror_ns2-test.status 2026-02-20T21:22:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror/ns2-test.status 2026-02-20T21:22:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror/ns2-test.status 2026-02-20T21:22:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:22:39.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns2-test.status 2026-02-20T21:22:39.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror/ns2 test /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns2-test.status 2026-02-20T21:22:39.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:39.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns2 2026-02-20T21:22:39.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:39.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror_ns2-test.status 2026-02-20T21:22:39.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/ns2/test --format xml 2026-02-20T21:22:39.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:22:40.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:22:40.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=193 2026-02-20T21:22:40.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:22:40.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:22:40.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:22:40.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns2 test /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns2-test.status 2026-02-20T21:22:40.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:40.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns2 2026-02-20T21:22:40.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:40.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns2-test.status 2026-02-20T21:22:40.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns2/test --format xml 2026-02-20T21:22:40.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:22:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:22:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=191 2026-02-20T21:22:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 193 = 191 2026-02-20T21:22:40.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:22:40.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.4 2026-02-20T21:22:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns2 test /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns2-test.status 2026-02-20T21:22:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns2 2026-02-20T21:22:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns2-test.status 2026-02-20T21:22:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns2/test --format xml 2026-02-20T21:22:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:22:40.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:22:40.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=191 2026-02-20T21:22:40.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 193 = 191 2026-02-20T21:22:40.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:22:40.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.8 2026-02-20T21:22:41.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns2 test /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns2-test.status 2026-02-20T21:22:41.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:41.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns2 2026-02-20T21:22:41.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:41.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns2-test.status 2026-02-20T21:22:41.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns2/test --format xml 2026-02-20T21:22:41.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:22:41.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:22:41.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=193 2026-02-20T21:22:41.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 193 = 193 2026-02-20T21:22:41.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:41.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror/ns1 test 2026-02-20T21:22:41.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:41.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:22:41.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:41.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:22:41.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror/ns1 test up+replaying 2026-02-20T21:22:41.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:41.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:22:41.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:41.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:22:41.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:22:41.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:22:41.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:22:41.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:22:42.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror/ns1 test up+replaying '' '' 2026-02-20T21:22:42.525 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:42.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:22:42.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:42.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:22:42.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:22:42.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:22:42.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:22:42.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:22:42.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/ns1/test 2026-02-20T21:22:42.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:22:42.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: cdfe5f4d-6ee5-4727-a7e9-c6361dbd5990 2026-02-20T21:22:42.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:22:42.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622553,"replay_state":"idle"} 2026-02-20T21:22:42.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:22:42.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:22:35 2026-02-20T21:22:42.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:22:42.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:22:42.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: down+unknown 2026-02-20T21:22:42.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: status not found 2026-02-20T21:22:42.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: ' 2026-02-20T21:22:42.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:22:42.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:22:42.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:22:42.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622553,"replay_state":"idle"} 2026-02-20T21:22:42.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:22:42.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:22:42.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:22:42.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:22:42.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:22:42.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:22:42.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:22:42.564 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:22:42.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:22:35' 2026-02-20T21:22:42.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror/ns1 test up+replaying '2026-02-20 21:22:35' 2026-02-20T21:22:42.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:42.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:22:42.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:42.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:22:42.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:22:35' 2026-02-20T21:22:42.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:22:42.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:22:42.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror/ns1 --verbose --format xml 2026-02-20T21:22:42.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING158775882mirror.1trial19920.2.0-677-gf7870454trueOK58845890mirror.2trial19920.2.0-677-gf7870454falseOK58915899mirror.3trial19920.2.0-677-gf7870454falseOK59065912mirror.0trial19920.2.0-677-gf7870454falseOKtestcdfe5f4d-6ee5-4727-a7e9-c6361dbd5990up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622553,"replay_state":"idle"}58775882mirror.1trial1992026-02-20 21:22:35cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8down+unknownstatus not found' 2026-02-20T21:22:42.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:22:42.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:22:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:22:35' 2026-02-20T21:22:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:22:42.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:22:42.601 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:22:42.601 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:22:42.602 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:22:42.603 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:42.603 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:42.603 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror/ns2 test 2026-02-20T21:22:42.603 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:42.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns2 2026-02-20T21:22:42.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:42.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:22:42.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror/ns2 test up+replaying 2026-02-20T21:22:42.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:42.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns2 2026-02-20T21:22:42.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:42.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:22:42.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:22:42.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:22:42.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:22:42.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:22:43.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror/ns2 test up+replaying '' '' 2026-02-20T21:22:43.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:43.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns2 2026-02-20T21:22:43.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:43.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:22:43.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:22:43.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:22:43.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:22:43.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:22:43.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/ns2/test 2026-02-20T21:22:43.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:22:43.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 6c61272f-0aa4-4613-863f-a017aa5e6dd1 2026-02-20T21:22:43.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:22:43.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622554,"replay_state":"idle"} 2026-02-20T21:22:43.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:22:43.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:22:36 2026-02-20T21:22:43.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:22:43.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:22:43.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: down+unknown 2026-02-20T21:22:43.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: status not found 2026-02-20T21:22:43.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: ' 2026-02-20T21:22:43.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:22:43.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:22:43.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:22:43.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622554,"replay_state":"idle"} 2026-02-20T21:22:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:22:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:22:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:22:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:22:43.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:22:43.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:22:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:22:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:22:43.846 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:22:36' 2026-02-20T21:22:43.846 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror/ns2 test up+replaying '2026-02-20 21:22:36' 2026-02-20T21:22:43.846 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:43.846 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns2 2026-02-20T21:22:43.846 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:43.846 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:22:43.847 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:22:36' 2026-02-20T21:22:43.847 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:22:43.847 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:22:43.847 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror/ns2 --verbose --format xml 2026-02-20T21:22:43.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING158775882mirror.1trial19920.2.0-677-gf7870454trueOK58845890mirror.2trial19920.2.0-677-gf7870454falseOK58915899mirror.3trial19920.2.0-677-gf7870454falseOK59065912mirror.0trial19920.2.0-677-gf7870454falseOKtest6c61272f-0aa4-4613-863f-a017aa5e6dd1up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622554,"replay_state":"idle"}58775882mirror.1trial1992026-02-20 21:22:36cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8down+unknownstatus not found' 2026-02-20T21:22:43.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:22:43.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:22:43.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:22:36' 2026-02-20T21:22:43.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:22:43.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:22:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:22:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:22:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:22:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror/ns1 mirror/ns1 test 2026-02-20T21:22:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:22:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:22:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror/ns1 2026-02-20T21:22:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror/ns1 2026-02-20T21:22:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror/ns1-test.export 2026-02-20T21:22:43.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror/ns1-test.export 2026-02-20T21:22:43.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:22:43.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test.export 2026-02-20T21:22:43.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror/ns1-test.export 2026-02-20T21:22:43.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror/ns1-test.export 2026-02-20T21:22:43.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:22:43.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.export 2026-02-20T21:22:43.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.export 2026-02-20T21:22:43.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/ns1/test /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test.export 2026-02-20T21:22:43.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:22:43.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/ns1/test /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.export 2026-02-20T21:22:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:22:44.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.export 2026-02-20T21:22:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test.export 2026-02-20T21:22:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror/ns2 mirror/ns2 test 2026-02-20T21:22:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:22:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:22:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror/ns2 2026-02-20T21:22:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror/ns2 2026-02-20T21:22:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror/ns2-test.export 2026-02-20T21:22:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror/ns2-test.export 2026-02-20T21:22:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:22:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror_ns2-test.export 2026-02-20T21:22:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror/ns2-test.export 2026-02-20T21:22:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror/ns2-test.export 2026-02-20T21:22:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:22:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns2-test.export 2026-02-20T21:22:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns2-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns2-test.export 2026-02-20T21:22:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/ns2/test /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns2-test.export 2026-02-20T21:22:44.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:22:44.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/ns2/test /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns2-test.export 2026-02-20T21:22:44.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:22:44.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns2-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns2-test.export 2026-02-20T21:22:44.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns2-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns2-test.export 2026-02-20T21:22:44.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:44.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog ' - disable mirroring / delete image' 2026-02-20T21:22:44.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:22:44.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:22:44.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:22:44 - disable mirroring / delete image 2026-02-20T21:22:44.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:22:44 - disable mirroring / delete image 2026-02-20T21:22:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster2 mirror/ns1 test 2026-02-20T21:22:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:22:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:22:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:22:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster2 mirror/ns1 test 2026-02-20T21:22:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:22:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 snap purge mirror/ns1/test 2026-02-20T21:22:44.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rm mirror/ns1/test 2026-02-20T21:22:46.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:22:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ disable_mirror cluster2 mirror/ns2 test 2026-02-20T21:22:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns2 2026-02-20T21:22:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/ns2/test 2026-02-20T21:22:47.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring disabled 2026-02-20T21:22:48.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror/ns1 test deleted 2026-02-20T21:22:48.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:48.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:22:48.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=deleted 2026-02-20T21:22:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:22:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:22:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror/ns1 test 2026-02-20T21:22:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror/ns1 2026-02-20T21:22:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:22:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/ns1/test 2026-02-20T21:22:48.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:22:48.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:22:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image_id= 2026-02-20T21:22:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:22:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:22:48.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror/ns1 test deleted '' 2026-02-20T21:22:48.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:48.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:22:48.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:48.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=deleted 2026-02-20T21:22:48.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:22:48.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:22:48.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:22:48.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror/ns1 test 2026-02-20T21:22:48.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:48.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror/ns1 2026-02-20T21:22:48.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:22:48.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/ns1/test 2026-02-20T21:22:48.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:22:48.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:22:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id= 2026-02-20T21:22:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:48.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test deleted = deleted 2026-02-20T21:22:48.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:48.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror/ns2 test deleted 2026-02-20T21:22:48.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:48.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns2 2026-02-20T21:22:48.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:48.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=deleted 2026-02-20T21:22:48.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:22:48.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:22:48.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:48.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror/ns2 test 2026-02-20T21:22:48.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:48.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror/ns2 2026-02-20T21:22:48.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:22:48.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/ns2/test 2026-02-20T21:22:48.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:22:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:22:48.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image_id= 2026-02-20T21:22:48.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:22:48.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:22:48.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror/ns2 test deleted '' 2026-02-20T21:22:48.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:48.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns2 2026-02-20T21:22:48.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:48.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=deleted 2026-02-20T21:22:48.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:22:48.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:22:48.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:22:48.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror/ns2 test 2026-02-20T21:22:48.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:48.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror/ns2 2026-02-20T21:22:48.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:22:48.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/ns2/test 2026-02-20T21:22:48.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:22:48.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:22:48.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id= 2026-02-20T21:22:48.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test deleted = deleted 2026-02-20T21:22:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster2 mirror/ns2 test 2026-02-20T21:22:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns2 2026-02-20T21:22:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:22:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:22:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster2 mirror/ns2 test 2026-02-20T21:22:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns2 2026-02-20T21:22:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:48.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 snap purge mirror/ns2/test 2026-02-20T21:22:48.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rm mirror/ns2/test 2026-02-20T21:22:48.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:22:48.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:48.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: mirror to a different remote namespace' 2026-02-20T21:22:48.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:22:48.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:22:48.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:22:48 TEST: mirror to a different remote namespace 2026-02-20T21:22:48.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:22:48 TEST: mirror to a different remote namespace 2026-02-20T21:22:48.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog ' - replay' 2026-02-20T21:22:48.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:22:48.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:22:48.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:22:48 - replay 2026-02-20T21:22:48.394 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:22:48 - replay 2026-02-20T21:22:48.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ NS3=ns3 2026-02-20T21:22:48.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ NS4=ns4 2026-02-20T21:22:48.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 namespace create mirror/ns3 2026-02-20T21:22:48.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 namespace create mirror/ns4 2026-02-20T21:22:48.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns3 image --remote-namespace ns4 2026-02-20T21:22:48.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns4 image --remote-namespace ns3 2026-02-20T21:22:48.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image_and_enable_mirror cluster2 mirror/ns4 test snapshot 2026-02-20T21:22:48.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:48.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:48.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns4 2026-02-20T21:22:48.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:48.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:48.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:48.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:22:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:22:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image cluster2 mirror/ns4 test 2026-02-20T21:22:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns4 2026-02-20T21:22:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=128 2026-02-20T21:22:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:22:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,object-map,fast-diff mirror/ns4/test 2026-02-20T21:22:48.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror/ns4 2026-02-20T21:22:48.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns4 2026-02-20T21:22:48.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:22:48.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:48.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror/ns4 test snapshot 2026-02-20T21:22:48.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:48.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns4 2026-02-20T21:22:48.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:48.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:22:48.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/ns4/test snapshot 2026-02-20T21:22:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:22:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/ns4/test 2026-02-20T21:22:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test': 2026-02-20T21:22:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:22:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:22:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:22:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 1828975ca8cc 2026-02-20T21:22:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.1828975ca8cc 2026-02-20T21:22:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:22:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:22:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:22:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:22:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:22:48 2026 2026-02-20T21:22:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:22:48 2026 2026-02-20T21:22:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:22:48 2026 2026-02-20T21:22:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:22:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:22:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 98612f61-4ab4-45ea-b329-982087e8aeba 2026-02-20T21:22:49.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:22:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror/ns3 test 2026-02-20T21:22:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:22:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror/ns3 test started 2026-02-20T21:22:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:22:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:22:49.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:22:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:22:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:22:50.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror/ns3 test started 2026-02-20T21:22:50.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:50.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:22:50.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:50.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:22:50.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:22:50.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:22:50.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/ns3/test 2026-02-20T21:22:50.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:22:50.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:22:50.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:50.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:50.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:22:50.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:22:50.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:22:50.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:22:50.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:22:50.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:22:50.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:50.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns3/test 2026-02-20T21:22:50.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:22:50.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:50.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:50.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:22:50.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:22:50.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:50.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:50.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:50.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:22:50.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:22:50.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:22:50.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:50.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:22:50.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:22:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:22:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:22:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:22:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:22:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:22:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:22:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:22:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:22:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:22:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:22:50.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:22:50.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:22:50.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:22:50.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:22:50.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:22:50.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:22:50.047 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns3/test 2026-02-20T21:22:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-02-20T21:22:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:22:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:22:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns3/test 2026-02-20T21:22:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:22:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:22:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:22:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:22:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:22:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:22:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:22:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:22:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:22:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:22:50.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:22:50.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:50.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:22:50.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:22:50.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:22:50.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:50.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:50.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:22:50.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:22:50.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:22:50.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:22:50.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:22:50.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:22:50.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:22:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:22:50.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:22:50.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:22:50.101 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns3/test 2026-02-20T21:22:50.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-02-20T21:22:50.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:22:50.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:22:50.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:50.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns3/test 2026-02-20T21:22:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:22:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:22:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:22:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:22:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:22:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:22:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:22:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:22:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:22:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:22:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:22:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:22:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:22:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:22:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:22:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:22:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:22:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:22:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:22:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:22:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:22:50.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:22:50.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:22:50.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:22:50.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns3/test 2026-02-20T21:22:50.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-02-20T21:22:50.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:22:50.195 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:22:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns3/test 2026-02-20T21:22:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:22:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:22:50.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:22:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:22:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:22:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:22:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:22:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:22:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:22:50.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:22:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:22:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:22:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:22:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:22:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:22:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:22:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:22:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:22:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:22:50.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:22:50.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:22:50.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:22:50.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:22:50.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:22:50.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns3/test 2026-02-20T21:22:50.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-02-20T21:22:50.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:22:50.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:22:50.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:22:50.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 1 2026-02-20T21:22:51.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:22:51.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:22:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns3/test 2026-02-20T21:22:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:22:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:22:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:22:51.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:22:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:22:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:22:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:22:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:22:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:22:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:22:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:22:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:22:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:22:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:22:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:22:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:22:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:22:51.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:22:51.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:22:51.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:22:51.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:22:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:22:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:22:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:22:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns3/test 2026-02-20T21:22:51.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-02-20T21:22:51.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:22:51.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:22:51.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:51.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns3/test 2026-02-20T21:22:51.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:22:51.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:51.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:51.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:22:51.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:22:51.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:51.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:51.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:51.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:22:51.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:22:51.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:22:51.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:51.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:22:51.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:22:51.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:22:51.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:22:51.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:22:51.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:51.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:22:51.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:22:51.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:22:51.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:51.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:51.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:22:51.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:22:51.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:22:51.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:22:51.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:22:51.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:22:51.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:22:51.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:22:51.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:22:51.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:22:51.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns3/test 2026-02-20T21:22:51.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-02-20T21:22:51.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:22:51.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:22:51.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:51.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns3/test 2026-02-20T21:22:51.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:22:51.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:51.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:51.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:22:51.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:22:51.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:51.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:51.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:51.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:22:51.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:22:51.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:22:51.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:51.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:22:51.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:22:51.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:22:51.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:22:51.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:22:51.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:51.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:22:51.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:22:51.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:22:51.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:51.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:51.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:22:51.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:22:51.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:22:51.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:22:51.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:22:51.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:22:51.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:22:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:22:51.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:22:51.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:22:51.374 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns3/test 2026-02-20T21:22:51.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-02-20T21:22:51.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:22:51.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:22:51.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:51.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns3/test 2026-02-20T21:22:51.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:22:51.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:51.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:51.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:22:51.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:22:51.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:51.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:51.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:51.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:22:51.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:22:51.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:22:51.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:51.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:22:51.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:22:51.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:22:51.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:22:51.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:22:51.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:51.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:22:51.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:22:51.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:22:51.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:51.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:51.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:22:51.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:22:51.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:22:51.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:22:51.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:22:51.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:22:51.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:22:51.428 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:22:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:22:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:22:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns3/test 2026-02-20T21:22:51.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-02-20T21:22:51.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:22:51.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:22:51.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:22:51.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 2 2026-02-20T21:22:53.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:22:53.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:22:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns3/test 2026-02-20T21:22:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:22:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:22:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:22:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:22:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:22:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:22:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:22:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:22:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:22:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:22:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:22:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:22:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:22:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:22:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:22:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:22:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:22:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:22:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:22:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:22:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:22:53.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:22:53.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:22:53.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:22:53.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns3/test 2026-02-20T21:22:53.526 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-02-20T21:22:53.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:22:53.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:22:53.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:22:53.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns3/test 2026-02-20T21:22:53.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:22:53.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:22:53.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:22:53.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:22:53.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:22:53.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:22:53.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:22:53.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:22:53.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:22:53.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:22:53.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:22:53.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:53.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:22:53.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:22:53.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:22:53.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:22:53.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:22:53.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:22:53.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:22:53.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:22:53.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:22:53.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:22:53.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:22:53.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:22:53.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:22:53.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:22:53.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:22:53.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:22:53.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:22:53.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:22:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:22:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:22:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:22:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns3/test 2026-02-20T21:22:53.585 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:22:53.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:22:53.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:22:53.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:22:53.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:22:53.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:22:53.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:22:53.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:53.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror/ns4 test 100 2026-02-20T21:22:53.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:53.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns4 2026-02-20T21:22:53.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:53.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:22:53.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:22:53.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:53.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:22:53.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/ns4/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:22:53.618 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:22:53.637 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:22:53.638 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5263.24 bytes/sec: 21 MiB/s 2026-02-20T21:22:53.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror/ns3 mirror/ns4 test 2026-02-20T21:22:53.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:22:53.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:53.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror/ns3 2026-02-20T21:22:53.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror/ns4 2026-02-20T21:22:53.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:53.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:22:53.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:22:53.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror/ns4 test 2026-02-20T21:22:53.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:53.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns4 2026-02-20T21:22:53.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:53.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/ns4/test 2026-02-20T21:22:54.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 195 2026-02-20T21:22:54.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror/ns3 mirror/ns4 test 2026-02-20T21:22:54.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:22:54.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:54.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror/ns3 2026-02-20T21:22:54.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror/ns4 2026-02-20T21:22:54.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:54.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror/ns4-test.status 2026-02-20T21:22:54.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror/ns4-test.status 2026-02-20T21:22:54.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:22:54.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror_ns4-test.status 2026-02-20T21:22:54.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror/ns3-test.status 2026-02-20T21:22:54.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror/ns3-test.status 2026-02-20T21:22:54.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:22:54.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.status 2026-02-20T21:22:54.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror/ns4 test /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns4-test.status 2026-02-20T21:22:54.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:54.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns4 2026-02-20T21:22:54.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:54.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror_ns4-test.status 2026-02-20T21:22:54.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/ns4/test --format xml 2026-02-20T21:22:54.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:22:54.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:22:54.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=195 2026-02-20T21:22:54.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:22:54.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:22:54.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:22:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns3 test /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.status 2026-02-20T21:22:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:22:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.status 2026-02-20T21:22:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns3/test --format xml 2026-02-20T21:22:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:22:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:22:54.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=194 2026-02-20T21:22:54.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 195 = 194 2026-02-20T21:22:54.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:22:54.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.4 2026-02-20T21:22:54.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns3 test /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.status 2026-02-20T21:22:54.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:54.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:22:54.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:54.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.status 2026-02-20T21:22:54.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns3/test --format xml 2026-02-20T21:22:54.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:22:54.869 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:22:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=194 2026-02-20T21:22:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 195 = 194 2026-02-20T21:22:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:22:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.8 2026-02-20T21:22:55.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns3 test /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.status 2026-02-20T21:22:55.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:55.672 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:22:55.672 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:55.672 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.status 2026-02-20T21:22:55.672 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns3/test --format xml 2026-02-20T21:22:55.672 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:22:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:22:55.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=195 2026-02-20T21:22:55.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 195 = 195 2026-02-20T21:22:55.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:55.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror/ns3 test 2026-02-20T21:22:55.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:55.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:22:55.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:55.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:22:55.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror/ns3 test up+replaying 2026-02-20T21:22:55.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:55.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:22:55.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:55.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:22:55.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:22:55.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:22:55.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:22:55.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:22:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror/ns3 test up+replaying '' '' 2026-02-20T21:22:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:22:56.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:56.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:22:56.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:22:56.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:22:56.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:22:56.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:22:56.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/ns3/test 2026-02-20T21:22:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:22:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 98612f61-4ab4-45ea-b329-982087e8aeba 2026-02-20T21:22:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:22:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622568,"replay_state":"idle"} 2026-02-20T21:22:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:22:56.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:22:53 2026-02-20T21:22:56.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:22:56.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:22:56.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: down+unknown 2026-02-20T21:22:56.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: status not found 2026-02-20T21:22:56.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: ' 2026-02-20T21:22:56.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:22:56.742 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:22:56.742 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:22:56.742 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622568,"replay_state":"idle"} 2026-02-20T21:22:56.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:22:56.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:22:56.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:22:56.743 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:22:56.744 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:22:56.744 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:22:56.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:22:56.745 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:22:56.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:22:53' 2026-02-20T21:22:56.746 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror/ns3 test up+replaying '2026-02-20 21:22:53' 2026-02-20T21:22:56.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:56.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:22:56.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:56.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:22:56.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:22:53' 2026-02-20T21:22:56.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:22:56.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:22:56.747 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror/ns3 --verbose --format xml 2026-02-20T21:22:56.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING158775882mirror.1trial19920.2.0-677-gf7870454trueOK58845890mirror.2trial19920.2.0-677-gf7870454falseOK58915899mirror.3trial19920.2.0-677-gf7870454falseOK59065912mirror.0trial19920.2.0-677-gf7870454falseOKtest98612f61-4ab4-45ea-b329-982087e8aebaup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622568,"replay_state":"idle"}58775882mirror.1trial1992026-02-20 21:22:53cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8down+unknownstatus not found' 2026-02-20T21:22:56.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:22:56.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:22:56.779 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:22:53' 2026-02-20T21:22:56.779 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:22:56.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:22:56.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:22:56.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:22:56.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:22:56.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:56.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:56.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror/ns3 mirror/ns4 test 2026-02-20T21:22:56.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:22:56.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:22:56.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:56.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror/ns3 2026-02-20T21:22:56.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror/ns4 2026-02-20T21:22:56.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:56.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror/ns4-test.export 2026-02-20T21:22:56.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror/ns4-test.export 2026-02-20T21:22:56.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:22:56.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror_ns4-test.export 2026-02-20T21:22:56.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror/ns3-test.export 2026-02-20T21:22:56.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror/ns3-test.export 2026-02-20T21:22:56.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:22:56.786 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.export 2026-02-20T21:22:56.786 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns4-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.export 2026-02-20T21:22:56.786 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/ns4/test /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns4-test.export 2026-02-20T21:22:56.871 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:22:56.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/ns3/test /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.export 2026-02-20T21:22:56.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:22:56.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns4-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.export 2026-02-20T21:22:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns4-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.export 2026-02-20T21:22:57.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:57.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog ' - disable mirroring and re-enable without remote-namespace' 2026-02-20T21:22:57.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:22:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:22:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:22:57 - disable mirroring and re-enable without remote-namespace 2026-02-20T21:22:57.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:22:57 - disable mirroring and re-enable without remote-namespace 2026-02-20T21:22:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster2 mirror/ns4 test 2026-02-20T21:22:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns4 2026-02-20T21:22:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:22:57.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:22:57.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster2 mirror/ns4 test 2026-02-20T21:22:57.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:57.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns4 2026-02-20T21:22:57.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:57.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 snap purge mirror/ns4/test 2026-02-20T21:22:57.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rm mirror/ns4/test 2026-02-20T21:22:58.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:22:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror/ns3 test deleted 2026-02-20T21:22:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:22:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=deleted 2026-02-20T21:22:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:22:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:22:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror/ns3 test 2026-02-20T21:22:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror/ns3 2026-02-20T21:22:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:22:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/ns3/test 2026-02-20T21:22:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:22:58.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:22:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image_id= 2026-02-20T21:22:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:22:58.212 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:22:58.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror/ns3 test deleted '' 2026-02-20T21:22:58.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:58.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:22:58.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:58.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=deleted 2026-02-20T21:22:58.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:22:58.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:22:58.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:22:58.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror/ns3 test 2026-02-20T21:22:58.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:22:58.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror/ns3 2026-02-20T21:22:58.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:22:58.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/ns3/test 2026-02-20T21:22:58.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:22:58.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:22:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id= 2026-02-20T21:22:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:22:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test deleted = deleted 2026-02-20T21:22:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror pool disable mirror/ns3 2026-02-20T21:22:58.366 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror pool disable mirror/ns4 2026-02-20T21:22:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 namespace create mirror/ns3 2026-02-20T21:22:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns3 image 2026-02-20T21:22:58.640 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns3 image 2026-02-20T21:22:58.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image_and_enable_mirror cluster2 mirror/ns3 test snapshot 2026-02-20T21:22:58.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:58.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:58.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:22:58.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:58.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:58.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:58.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:22:58.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:22:58.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:58.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image cluster2 mirror/ns3 test 2026-02-20T21:22:58.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:58.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:58.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:22:58.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:58.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:58.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:22:58.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=128 2026-02-20T21:22:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:22:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,object-map,fast-diff mirror/ns3/test 2026-02-20T21:22:58.695 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror/ns3 2026-02-20T21:22:58.695 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:22:58.695 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:22:58.695 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:22:58.695 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror/ns3 test snapshot 2026-02-20T21:22:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:22:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:22:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:22:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/ns3/test snapshot 2026-02-20T21:22:59.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:22:59.171 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/ns3/test 2026-02-20T21:22:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test': 2026-02-20T21:22:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:22:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:22:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:22:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 184c7ce9db6a 2026-02-20T21:22:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.184c7ce9db6a 2026-02-20T21:22:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:22:59.196 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:22:59.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:22:59.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:22:59.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:22:58 2026 2026-02-20T21:22:59.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:22:58 2026 2026-02-20T21:22:59.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:22:58 2026 2026-02-20T21:22:59.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:22:59.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:22:59.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 6e53ee34-02ee-4805-bed1-14b5ec5cddb0 2026-02-20T21:22:59.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:22:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror/ns3 test 2026-02-20T21:22:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:22:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror/ns3 test started 2026-02-20T21:22:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:22:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:22:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:22:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:22:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:22:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:22:59.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:23:00.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror/ns3 test started 2026-02-20T21:23:00.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:00.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:23:00.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:00.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:23:00.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:23:00.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:23:00.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/ns3/test 2026-02-20T21:23:00.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:23:00.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:23:00.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:23:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:23:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:23:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:23:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:23:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns3/test 2026-02-20T21:23:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:23:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:23:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:23:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:23:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:23:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:23:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:23:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:23:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:23:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:23:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:23:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:23:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns3/test 2026-02-20T21:23:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-02-20T21:23:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:23:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:23:00.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:00.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns3/test 2026-02-20T21:23:00.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:23:00.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:00.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:00.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:23:00.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:23:00.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:00.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:23:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:23:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:23:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:23:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:23:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:23:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:23:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:23:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:23:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:23:00.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:23:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:00.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:00.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns3/test 2026-02-20T21:23:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-02-20T21:23:00.311 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:23:00.312 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:23:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns3/test 2026-02-20T21:23:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:23:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:23:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:23:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:23:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:23:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:23:00.315 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:23:00.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:23:00.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:23:00.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:00.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:00.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:23:00.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:23:00.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:00.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:00.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:00.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:23:00.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:23:00.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:00.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:23:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:23:00.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:23:00.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:23:00.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:23:00.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:23:00.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns3/test 2026-02-20T21:23:00.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-02-20T21:23:00.364 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:23:00.365 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:23:00.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:00.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns3/test 2026-02-20T21:23:00.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:23:00.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:00.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:00.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:23:00.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:23:00.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:00.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:00.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:00.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:23:00.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:23:00.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:00.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:00.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:23:00.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:23:00.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:23:00.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:23:00.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:00.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:00.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:23:00.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:23:00.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:00.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:00.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:00.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:23:00.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:23:00.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:00.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:00.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:23:00.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:23:00.370 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:23:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:23:00.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:23:00.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:23:00.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns3/test 2026-02-20T21:23:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-02-20T21:23:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:23:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:23:00.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:00.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 1 2026-02-20T21:23:01.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:23:01.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:23:01.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:01.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns3/test 2026-02-20T21:23:01.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:23:01.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:01.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:01.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:23:01.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:23:01.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:01.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:01.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:01.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:23:01.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:01.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:01.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:01.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:23:01.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:01.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:23:01.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:23:01.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:01.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:01.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:23:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:23:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:23:01.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:23:01.428 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:23:01.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:01.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:01.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:01.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns3/test 2026-02-20T21:23:01.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-02-20T21:23:01.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:23:01.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:23:01.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns3/test 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:23:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:23:01.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:23:01.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:01.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:01.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:01.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns3/test 2026-02-20T21:23:01.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-02-20T21:23:01.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:23:01.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns3/test 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:23:01.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:23:01.536 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:23:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:23:01.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:23:01.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:23:01.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns3/test 2026-02-20T21:23:01.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-02-20T21:23:01.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:23:01.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:23:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns3/test 2026-02-20T21:23:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:23:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:23:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:23:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:23:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:23:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:23:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:23:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:23:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:23:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:23:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:23:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:23:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:23:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:23:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:23:01.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:23:01.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:23:01.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:23:01.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:23:01.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns3/test 2026-02-20T21:23:01.641 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-02-20T21:23:01.641 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:23:01.642 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:23:01.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:01.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 2 2026-02-20T21:23:03.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:23:03.644 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:23:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:03.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns3/test 2026-02-20T21:23:03.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:23:03.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:03.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:03.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:23:03.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:23:03.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:03.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:03.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:03.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:23:03.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:03.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:03.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:03.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:23:03.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:03.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:23:03.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:23:03.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:03.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:03.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:23:03.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:23:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:23:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:23:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:23:03.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:03.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:03.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:03.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns3/test 2026-02-20T21:23:03.700 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns3/test'] 2026-02-20T21:23:03.700 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:23:03.701 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:23:03.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:03.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns3/test 2026-02-20T21:23:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:23:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:23:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:23:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:23:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:03.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:23:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:23:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:23:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:23:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:23:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:23:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:23:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:23:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:23:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:23:03.714 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:03.714 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:03.714 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:03.714 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns3/test 2026-02-20T21:23:03.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:23:03.761 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:23:03.762 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:23:03.762 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:23:03.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:23:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:23:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:23:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror/ns3 test 100 2026-02-20T21:23:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:23:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:03.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:23:03.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:23:03.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:23:03.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:23:03.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/ns3/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:23:03.795 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:23:03.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:23:03.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5555.64 bytes/sec: 22 MiB/s 2026-02-20T21:23:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror/ns3 mirror/ns3 test 2026-02-20T21:23:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:23:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror/ns3 2026-02-20T21:23:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror/ns3 2026-02-20T21:23:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:23:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:23:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror/ns3 test 2026-02-20T21:23:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:23:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/ns3/test 2026-02-20T21:23:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 197 2026-02-20T21:23:03.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror/ns3 mirror/ns3 test 2026-02-20T21:23:03.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:23:03.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:03.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror/ns3 2026-02-20T21:23:03.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror/ns3 2026-02-20T21:23:03.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:03.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror/ns3-test.status 2026-02-20T21:23:03.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror/ns3-test.status 2026-02-20T21:23:03.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:23:03.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror_ns3-test.status 2026-02-20T21:23:03.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror/ns3-test.status 2026-02-20T21:23:03.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror/ns3-test.status 2026-02-20T21:23:03.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:23:03.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.status 2026-02-20T21:23:03.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror/ns3 test /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns3-test.status 2026-02-20T21:23:03.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:03.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:23:03.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:03.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror_ns3-test.status 2026-02-20T21:23:03.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/ns3/test --format xml 2026-02-20T21:23:03.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:23:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:23:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=197 2026-02-20T21:23:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:23:04.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:23:04.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:23:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns3 test /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.status 2026-02-20T21:23:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:23:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.status 2026-02-20T21:23:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns3/test --format xml 2026-02-20T21:23:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:23:04.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:23:04.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=196 2026-02-20T21:23:04.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 197 = 196 2026-02-20T21:23:04.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:23:04.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.4 2026-02-20T21:23:04.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns3 test /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.status 2026-02-20T21:23:04.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:04.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:23:04.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:04.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.status 2026-02-20T21:23:04.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns3/test --format xml 2026-02-20T21:23:04.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:23:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:23:04.684 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=196 2026-02-20T21:23:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 197 = 196 2026-02-20T21:23:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:23:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.8 2026-02-20T21:23:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns3 test /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.status 2026-02-20T21:23:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:23:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.status 2026-02-20T21:23:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns3/test --format xml 2026-02-20T21:23:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:23:05.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:23:05.520 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=197 2026-02-20T21:23:05.520 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 197 = 197 2026-02-20T21:23:05.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:05.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror/ns3 test 2026-02-20T21:23:05.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:05.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:23:05.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:05.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:23:05.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror/ns3 test up+replaying 2026-02-20T21:23:05.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:05.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:23:05.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:05.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:23:05.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:23:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:23:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:23:06.521 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror/ns3 test up+replaying '' '' 2026-02-20T21:23:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:23:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:23:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:23:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:23:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:23:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:23:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/ns3/test 2026-02-20T21:23:06.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:23:06.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 6e53ee34-02ee-4805-bed1-14b5ec5cddb0 2026-02-20T21:23:06.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:23:06.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622579,"replay_state":"idle"} 2026-02-20T21:23:06.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:23:06.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:23:03 2026-02-20T21:23:06.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:23:06.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:23:06.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: down+unknown 2026-02-20T21:23:06.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: status not found 2026-02-20T21:23:06.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: ' 2026-02-20T21:23:06.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:23:06.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:23:06.554 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:23:06.554 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622579,"replay_state":"idle"} 2026-02-20T21:23:06.555 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:23:06.555 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:23:06.555 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:23:06.555 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:23:06.556 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:23:06.556 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:23:06.557 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:23:06.557 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:23:06.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:23:03' 2026-02-20T21:23:06.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror/ns3 test up+replaying '2026-02-20 21:23:03' 2026-02-20T21:23:06.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:06.558 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:23:06.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:06.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:23:06.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:23:03' 2026-02-20T21:23:06.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:23:06.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:23:06.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror/ns3 --verbose --format xml 2026-02-20T21:23:06.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING158775882mirror.1trial19920.2.0-677-gf7870454trueOK58845890mirror.2trial19920.2.0-677-gf7870454falseOK58915899mirror.3trial19920.2.0-677-gf7870454falseOK59065912mirror.0trial19920.2.0-677-gf7870454falseOKtest6e53ee34-02ee-4805-bed1-14b5ec5cddb0up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622579,"replay_state":"idle"}58775882mirror.1trial1992026-02-20 21:23:03cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8down+unknownstatus not found' 2026-02-20T21:23:06.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:23:06.589 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:23:06.590 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:23:03' 2026-02-20T21:23:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:23:06.592 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:23:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:23:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:23:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:23:06.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:06.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:06.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror/ns3 mirror/ns3 test 2026-02-20T21:23:06.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:23:06.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:23:06.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:06.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror/ns3 2026-02-20T21:23:06.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror/ns3 2026-02-20T21:23:06.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:06.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror/ns3-test.export 2026-02-20T21:23:06.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror/ns3-test.export 2026-02-20T21:23:06.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:23:06.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror_ns3-test.export 2026-02-20T21:23:06.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror/ns3-test.export 2026-02-20T21:23:06.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror/ns3-test.export 2026-02-20T21:23:06.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:23:06.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.export 2026-02-20T21:23:06.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns3-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.export 2026-02-20T21:23:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/ns3/test /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns3-test.export 2026-02-20T21:23:06.682 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:23:06.686 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/ns3/test /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.export 2026-02-20T21:23:06.773 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:23:06.777 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns3-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.export 2026-02-20T21:23:06.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns3-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns3-test.export 2026-02-20T21:23:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster2 mirror/ns3 test 2026-02-20T21:23:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:23:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:06.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:23:06.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:23:06.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster2 mirror/ns3 test 2026-02-20T21:23:06.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:06.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:23:06.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:06.862 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 snap purge mirror/ns3/test 2026-02-20T21:23:06.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rm mirror/ns3/test 2026-02-20T21:23:08.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:23:08.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:08.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror/ns3 test deleted 2026-02-20T21:23:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:23:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=deleted 2026-02-20T21:23:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:23:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:23:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:23:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror/ns3 test 2026-02-20T21:23:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror/ns3 2026-02-20T21:23:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:23:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/ns3/test 2026-02-20T21:23:08.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:23:08.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:23:08.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image_id= 2026-02-20T21:23:08.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:23:08.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:23:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror/ns3 test deleted '' 2026-02-20T21:23:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns3 2026-02-20T21:23:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=deleted 2026-02-20T21:23:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:23:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:23:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:23:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror/ns3 test 2026-02-20T21:23:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror/ns3 2026-02-20T21:23:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:23:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/ns3/test 2026-02-20T21:23:08.138 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:23:08.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:23:08.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id= 2026-02-20T21:23:08.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:23:08.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test deleted = deleted 2026-02-20T21:23:08.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:08.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 mirror pool disable mirror/ns3 2026-02-20T21:23:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror pool disable mirror/ns3 2026-02-20T21:23:08.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog ' - data pool' 2026-02-20T21:23:08.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:23:08.218 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:23:08.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:23:08 - data pool 2026-02-20T21:23:08.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:23:08 - data pool 2026-02-20T21:23:08.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ dp_image=test_data_pool 2026-02-20T21:23:08.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image_and_enable_mirror cluster2 mirror/ns1 test_data_pool snapshot 128 --data-pool mirror_parent 2026-02-20T21:23:08.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:08.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:23:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:23:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:23:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:23:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image cluster2 mirror/ns1 test_data_pool 128 --data-pool mirror_parent 2026-02-20T21:23:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:23:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:23:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=128 2026-02-20T21:23:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 128 ']' 2026-02-20T21:23:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=128 2026-02-20T21:23:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:08.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,object-map,fast-diff --data-pool mirror_parent mirror/ns1/test_data_pool 2026-02-20T21:23:08.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror/ns1 2026-02-20T21:23:08.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:23:08.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:23:08.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:08.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror/ns1 test_data_pool snapshot 2026-02-20T21:23:08.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:08.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:23:08.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:23:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:23:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/ns1/test_data_pool snapshot 2026-02-20T21:23:09.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:23:09.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/ns1/test_data_pool 2026-02-20T21:23:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test_data_pool': 2026-02-20T21:23:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:23:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:23:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:23:09.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 186a101eecd 2026-02-20T21:23:09.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: data_pool: mirror_parent 2026-02-20T21:23:09.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.3.186a101eecd 2026-02-20T21:23:09.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:23:09.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff, data-pool 2026-02-20T21:23:09.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:23:09.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:23:09.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:23:08 2026 2026-02-20T21:23:09.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:23:08 2026 2026-02-20T21:23:09.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:23:08 2026 2026-02-20T21:23:09.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:23:09.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:23:09.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 287d0563-3caa-475b-9974-7258297fcf41 2026-02-20T21:23:09.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:23:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_data_pool cluster2 mirror/ns1 test_data_pool 2026-02-20T21:23:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:23:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror/ns1 2026-02-20T21:23:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test_data_pool 2026-02-20T21:23:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 info mirror/ns1/test_data_pool 2026-02-20T21:23:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ awk '$1 == "data_pool:" {print $2}' 2026-02-20T21:23:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ data_pool=mirror_parent 2026-02-20T21:23:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test mirror_parent = mirror_parent 2026-02-20T21:23:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror/ns1 test_data_pool 2026-02-20T21:23:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:23:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:23:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror/ns1 test_data_pool started 2026-02-20T21:23:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:23:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:23:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:23:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:23:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:23:10.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror/ns1 test_data_pool started 2026-02-20T21:23:10.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:10.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:23:10.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:23:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:23:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:23:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:23:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/ns1/test_data_pool 2026-02-20T21:23:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:23:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:23:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:23:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:23:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:10.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:23:10.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:23:10.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:23:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test_data_pool 2026-02-20T21:23:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:23:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:23:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:23:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:23:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:23:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:23:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:23:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:23:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:23:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:23:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:23:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:23:10.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:10.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-20T21:23:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-02-20T21:23:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:23:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:23:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns1/test_data_pool 2026-02-20T21:23:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:23:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:23:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:23:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:23:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:23:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:23:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:23:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:23:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:23:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:23:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:23:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:23:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:23:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:23:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:10.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:10.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-20T21:23:10.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-02-20T21:23:10.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:23:10.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:23:10.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns1/test_data_pool 2026-02-20T21:23:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:23:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:23:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:23:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:23:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:23:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:23:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:23:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:23:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:23:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:23:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:23:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:23:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:23:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:23:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:23:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:23:10.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:23:10.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:23:10.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:23:10.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-20T21:23:10.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-02-20T21:23:10.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:23:10.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:23:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns1/test_data_pool 2026-02-20T21:23:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:23:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:23:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:23:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:10.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:23:10.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:23:10.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:10.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:10.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:23:10.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:23:10.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:23:10.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:23:10.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:10.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:10.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:23:10.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:23:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:23:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:23:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:23:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:23:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:23:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:23:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:23:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:23:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-20T21:23:10.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-02-20T21:23:10.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:23:10.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:23:10.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:10.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 1 2026-02-20T21:23:11.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:23:11.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:23:11.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:11.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test_data_pool 2026-02-20T21:23:11.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:23:11.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:11.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:11.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:23:11.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:23:11.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:11.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:11.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:11.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:23:11.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:11.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:11.442 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:11.443 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:23:11.443 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:11.443 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:23:11.443 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:23:11.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:11.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:11.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:23:11.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:23:11.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:11.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:11.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:11.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:11.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:11.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:11.446 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:11.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:23:11.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:23:11.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:23:11.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:11.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:11.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:11.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-20T21:23:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-02-20T21:23:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:23:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:23:11.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:11.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns1/test_data_pool 2026-02-20T21:23:11.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:23:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:23:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:23:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:23:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:23:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:23:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:23:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:23:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:11.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:23:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:23:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:23:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:23:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:23:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:23:11.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-20T21:23:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:23:11.554 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:23:11.554 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:23:11.554 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:23:11.555 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:23:11.555 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:23:11.555 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:23:11.555 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:11.555 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_data_pool cluster1 mirror/ns1 test_data_pool 2026-02-20T21:23:11.555 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:11.556 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror/ns1 2026-02-20T21:23:11.556 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test_data_pool 2026-02-20T21:23:11.556 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 info mirror/ns1/test_data_pool 2026-02-20T21:23:11.556 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ awk '$1 == "data_pool:" {print $2}' 2026-02-20T21:23:11.585 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ data_pool=mirror_parent 2026-02-20T21:23:11.585 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test mirror_parent = mirror_parent 2026-02-20T21:23:11.585 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror/ns1 test_data_pool 100 2026-02-20T21:23:11.585 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:11.585 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:23:11.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:23:11.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:23:11.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:23:11.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:23:11.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:23:11.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/ns1/test_data_pool --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:23:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:23:11.633 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:23:11.633 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 5882.44 bytes/sec: 23 MiB/s 2026-02-20T21:23:11.638 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror/ns1 mirror/ns1 test_data_pool 2026-02-20T21:23:11.638 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:23:11.638 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:11.638 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror/ns1 2026-02-20T21:23:11.638 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror/ns1 2026-02-20T21:23:11.638 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:23:11.638 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:23:11.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:23:11.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror/ns1 test_data_pool 2026-02-20T21:23:11.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:11.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:23:11.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:23:11.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/ns1/test_data_pool 2026-02-20T21:23:11.726 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 17 2026-02-20T21:23:11.732 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror/ns1 mirror/ns1 test_data_pool 2026-02-20T21:23:11.732 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:23:11.732 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:11.732 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror/ns1 2026-02-20T21:23:11.732 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror/ns1 2026-02-20T21:23:11.732 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:23:11.733 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror/ns1-test_data_pool.status 2026-02-20T21:23:11.733 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror/ns1-test_data_pool.status 2026-02-20T21:23:11.733 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:23:11.734 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test_data_pool.status 2026-02-20T21:23:11.734 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror/ns1-test_data_pool.status 2026-02-20T21:23:11.734 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror/ns1-test_data_pool.status 2026-02-20T21:23:11.734 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:23:11.735 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test_data_pool.status 2026-02-20T21:23:11.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror/ns1 test_data_pool /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test_data_pool.status 2026-02-20T21:23:11.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:11.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:23:11.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:23:11.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test_data_pool.status 2026-02-20T21:23:11.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/ns1/test_data_pool --format xml 2026-02-20T21:23:11.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:23:11.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:23:11.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=17 2026-02-20T21:23:11.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:23:11.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:23:11.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:23:11.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test_data_pool /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test_data_pool.status 2026-02-20T21:23:11.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:11.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:23:11.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:23:11.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test_data_pool.status 2026-02-20T21:23:11.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test_data_pool --format xml 2026-02-20T21:23:11.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:23:11.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:23:11.997 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:23:11.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:-:1.1: Document is empty 2026-02-20T21:23:11.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 2026-02-20T21:23:11.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:^ 2026-02-20T21:23:11.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id= 2026-02-20T21:23:11.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 17 = '' 2026-02-20T21:23:11.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:23:11.999 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.4 2026-02-20T21:23:12.401 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test_data_pool /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test_data_pool.status 2026-02-20T21:23:12.401 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:12.401 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:23:12.401 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:23:12.401 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test_data_pool.status 2026-02-20T21:23:12.401 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test_data_pool --format xml 2026-02-20T21:23:12.401 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:23:12.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:23:12.433 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=16 2026-02-20T21:23:12.433 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 17 = 16 2026-02-20T21:23:12.433 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:23:12.433 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.8 2026-02-20T21:23:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 test_data_pool /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test_data_pool.status 2026-02-20T21:23:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:23:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:23:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test_data_pool.status 2026-02-20T21:23:13.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/test_data_pool --format xml 2026-02-20T21:23:13.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:23:13.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:23:13.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=17 2026-02-20T21:23:13.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 17 = 17 2026-02-20T21:23:13.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:13.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror/ns1 test_data_pool 2026-02-20T21:23:13.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:13.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:23:13.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:23:13.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:23:13.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror/ns1 test_data_pool up+replaying 2026-02-20T21:23:13.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:13.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:23:13.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:23:13.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:23:13.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:23:13.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:23:13.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:13.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:23:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror/ns1 test_data_pool up+replaying '' '' 2026-02-20T21:23:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:23:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:23:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:23:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:23:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:23:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:23:14.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:23:14.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/ns1/test_data_pool 2026-02-20T21:23:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_data_pool: 2026-02-20T21:23:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 287d0563-3caa-475b-9974-7258297fcf41 2026-02-20T21:23:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:23:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622589,"replay_state":"idle"} 2026-02-20T21:23:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:23:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:23:11 2026-02-20T21:23:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:23:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:23:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: down+unknown 2026-02-20T21:23:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: status not found 2026-02-20T21:23:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: ' 2026-02-20T21:23:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:23:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:23:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:23:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622589,"replay_state":"idle"} 2026-02-20T21:23:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:23:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:23:14.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:23:14.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:23:14.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:23:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:23:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:23:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:23:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:23:11' 2026-02-20T21:23:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror/ns1 test_data_pool up+replaying '2026-02-20 21:23:11' 2026-02-20T21:23:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:23:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:23:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:23:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:23:11' 2026-02-20T21:23:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:23:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:23:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror/ns1 --verbose --format xml 2026-02-20T21:23:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING158775882mirror.1trial19920.2.0-677-gf7870454trueOK58845890mirror.2trial19920.2.0-677-gf7870454falseOK58915899mirror.3trial19920.2.0-677-gf7870454falseOK59065912mirror.0trial19920.2.0-677-gf7870454falseOKtest_data_pool287d0563-3caa-475b-9974-7258297fcf41up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622589,"replay_state":"idle"}58775882mirror.1trial1992026-02-20 21:23:11cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8down+unknownstatus not found' 2026-02-20T21:23:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:23:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_data_pool'\'']/last_update' 2026-02-20T21:23:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:23:11' 2026-02-20T21:23:14.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_data_pool'\'']/state' 2026-02-20T21:23:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:23:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:23:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:23:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:23:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror/ns1 mirror/ns1 test_data_pool 2026-02-20T21:23:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:23:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:23:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror/ns1 2026-02-20T21:23:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror/ns1 2026-02-20T21:23:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:23:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror/ns1-test_data_pool.export 2026-02-20T21:23:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror/ns1-test_data_pool.export 2026-02-20T21:23:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:23:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test_data_pool.export 2026-02-20T21:23:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror/ns1-test_data_pool.export 2026-02-20T21:23:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror/ns1-test_data_pool.export 2026-02-20T21:23:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:23:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test_data_pool.export 2026-02-20T21:23:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test_data_pool.export /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test_data_pool.export 2026-02-20T21:23:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/ns1/test_data_pool /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test_data_pool.export 2026-02-20T21:23:14.434 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:23:14.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/ns1/test_data_pool /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test_data_pool.export 2026-02-20T21:23:14.522 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:23:14.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test_data_pool.export /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test_data_pool.export 2026-02-20T21:23:14.576 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror_ns1-test_data_pool.export /tmp/tmp.2FjbyoErqN/cluster1-mirror_ns1-test_data_pool.export 2026-02-20T21:23:14.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:14.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster2 mirror/ns1 test_data_pool 2026-02-20T21:23:14.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:14.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:23:14.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:23:14.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:23:14.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:23:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster2 mirror/ns1 test_data_pool 2026-02-20T21:23:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror/ns1 2026-02-20T21:23:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_data_pool 2026-02-20T21:23:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 snap purge mirror/ns1/test_data_pool 2026-02-20T21:23:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rm mirror/ns1/test_data_pool 2026-02-20T21:23:16.186 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:23:16.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:16.188 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: simple image resync' 2026-02-20T21:23:16.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:23:16.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:23:16.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:23:16 TEST: simple image resync 2026-02-20T21:23:16.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:23:16 TEST: simple image resync 2026-02-20T21:23:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ request_resync_image cluster1 mirror test image_id 2026-02-20T21:23:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id_var_name=image_id 2026-02-20T21:23:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:23:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:23:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:23:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:23:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:23:16.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval 'image_id='\''16faa5eeddfc'\''' 2026-02-20T21:23:16.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ image_id=16faa5eeddfc 2026-02-20T21:23:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval 'test -n "$image_id"' 2026-02-20T21:23:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -n 16faa5eeddfc 2026-02-20T21:23:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/test 2026-02-20T21:23:16.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Flagged image for resync from primary 2026-02-20T21:23:16.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror test deleted 16faa5eeddfc 2026-02-20T21:23:16.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:16.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:16.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:16.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=deleted 2026-02-20T21:23:16.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id=16faa5eeddfc 2026-02-20T21:23:16.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:23:16.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n 16faa5eeddfc 2026-02-20T21:23:16.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:23:16.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:23:16.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test deleted 16faa5eeddfc 2026-02-20T21:23:16.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:16.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:16.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:16.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=deleted 2026-02-20T21:23:16.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id=16faa5eeddfc 2026-02-20T21:23:16.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:23:16.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:23:16.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:23:16.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:16.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:23:16.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:23:16.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:23:16.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:23:16.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:23:16.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id= 2026-02-20T21:23:16.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:23:16.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test deleted = deleted 2026-02-20T21:23:16.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:16.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror test present 2026-02-20T21:23:16.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:16.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:16.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:16.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=present 2026-02-20T21:23:16.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:23:16.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:23:16.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:23:16.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:23:16.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:16.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:23:16.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:23:16.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:23:16.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:23:16.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:23:16.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image_id= 2026-02-20T21:23:16.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:23:16.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:23:16.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-20T21:23:16.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=present 2026-02-20T21:23:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:23:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:23:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:23:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:23:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:23:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:23:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:23:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:23:16.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:23:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id= 2026-02-20T21:23:16.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:23:16.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test present = deleted 2026-02-20T21:23:16.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:23:16.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:23:17.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-20T21:23:17.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:17.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:17.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:17.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=present 2026-02-20T21:23:17.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:23:17.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:23:17.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:23:17.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:23:17.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:17.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:23:17.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:23:17.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:23:17.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:23:17.760 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:23:17.763 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id= 2026-02-20T21:23:17.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:23:17.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test present = deleted 2026-02-20T21:23:17.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:23:17.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:23:19.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-20T21:23:19.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:19.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:19.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:19.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=present 2026-02-20T21:23:19.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:23:19.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:23:19.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:23:19.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:23:19.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:19.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:23:19.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:23:19.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:23:19.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:23:19.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:23:19.792 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id= 2026-02-20T21:23:19.792 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:23:19.792 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test present = deleted 2026-02-20T21:23:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:23:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:23:23.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-20T21:23:23.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:23.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:23.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:23.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=present 2026-02-20T21:23:23.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:23:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:23:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:23:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:23:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:23:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:23:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:23:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:23:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id=16faab1bc6b8 2026-02-20T21:23:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n 16faab1bc6b8 2026-02-20T21:23:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -z '' -o '' = 16faab1bc6b8 2026-02-20T21:23:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=present 2026-02-20T21:23:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test present = present 2026-02-20T21:23:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:23:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:23:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:23:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:23:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:23:24.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:23:24.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:24.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:24.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:24.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:23:24.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:23:24.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:23:24.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:23:24.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:23:24.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:23:24.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:24.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:24.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:23:24.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:23:24.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:24.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:23:24.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:23:24.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:23:24.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:23:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:23:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:23:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:23:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:23:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:23:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:23:24.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:23:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:23:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:23:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:23:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:23:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:23:24.843 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:24.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:24.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:24.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:23:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:23:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:23:24.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:23:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:24.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:23:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:23:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:23:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:23:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:23:24.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:23:24.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:23:24.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:23:24.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:23:24.945 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:23:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:23:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:23:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-20T21:23:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:23:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:23:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:23:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:23:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:23:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:24.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:23:25.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:23:25.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:25.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:25.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:25.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:23:25.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:23:25.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:23:25.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:23:25.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:23:25.949 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:23:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:23:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: d5138118-365e-4c80-9761-89b69f10c700 2026-02-20T21:23:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:23:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622549,"replay_state":"idle"} 2026-02-20T21:23:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:23:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:23:21 2026-02-20T21:23:25.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:23:25.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:23:25.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: down+unknown 2026-02-20T21:23:25.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: status not found 2026-02-20T21:23:25.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: ' 2026-02-20T21:23:25.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:23:25.980 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:23:25.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:23:25.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622549,"replay_state":"idle"} 2026-02-20T21:23:25.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:23:25.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:23:25.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:23:25.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:23:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:23:25.983 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:23:25.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:23:25.984 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:23:25.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:23:21' 2026-02-20T21:23:25.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:23:21' 2026-02-20T21:23:25.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:25.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:23:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:23:21' 2026-02-20T21:23:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:23:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:23:25.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:23:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING158775882mirror.1trial19920.2.0-677-gf7870454trueOK58845890mirror.2trial19920.2.0-677-gf7870454falseOK58915899mirror.3trial19920.2.0-677-gf7870454falseOK59065912mirror.0trial19920.2.0-677-gf7870454falseOKtestd5138118-365e-4c80-9761-89b69f10c700up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622549,"replay_state":"idle"}58775882mirror.1trial1992026-02-20 21:23:21cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8down+unknownstatus not found' 2026-02-20T21:23:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:23:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:23:26.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:23:21' 2026-02-20T21:23:26.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:23:26.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:23:26.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:23:26.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:23:26.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:23:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror mirror test 2026-02-20T21:23:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:23:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:23:26.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:26.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:23:26.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:23:26.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:26.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.export 2026-02-20T21:23:26.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.export 2026-02-20T21:23:26.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:23:26.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export 2026-02-20T21:23:26.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.export 2026-02-20T21:23:26.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.export 2026-02-20T21:23:26.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:23:26.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:23:26.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:23:26.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export 2026-02-20T21:23:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:23:26.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:23:26.211 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:23:26.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:23:26.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:23:26.295 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:26.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -z '' ']' 2026-02-20T21:23:26.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: image resync while replayer is stopped' 2026-02-20T21:23:26.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:23:26.296 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:23:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:23:26 TEST: image resync while replayer is stopped 2026-02-20T21:23:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:23:26 TEST: image resync while replayer is stopped 2026-02-20T21:23:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemons cluster1 rbd mirror stop mirror/test 2026-02-20T21:23:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_instance=cluster1 2026-02-20T21:23:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance=cluster1 2026-02-20T21:23:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loop_instance 2026-02-20T21:23:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:23:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:23:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:23:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:0 rbd mirror stop mirror/test 2026-02-20T21:23:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:23:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:23:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:23:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:23:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:23:26.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:23:26.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:23:26.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:23:26.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:26.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:23:26.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:26.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:23:26.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:23:26.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:23:26.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:23:26.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:26.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:26.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:23:26.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:23:26.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:23:26.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:26.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:26.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:23:26.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:26.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:23:26.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:23:26.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:23:26.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:26.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:23:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:26.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:26.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:26.314 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop mirror/test 2026-02-20T21:23:26.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:23:26.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror stop : stop rbd mirror 2026-02-20T21:23:26.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:23:26.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:26.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:1 rbd mirror stop mirror/test 2026-02-20T21:23:26.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:23:26.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:26.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:23:26.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:23:26.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:23:26.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:23:26.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:23:26.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:23:26.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:23:26.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:26.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:23:26.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:26.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:23:26.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:23:26.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:23:26.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:23:26.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:26.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:26.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:23:26.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:23:26.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:23:26.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:26.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:26.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:23:26.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:26.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:23:26.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:23:26.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:23:26.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:23:26.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:23:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror stop mirror/test 2026-02-20T21:23:26.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:26.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-20T21:23:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-20T21:23:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:23:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:23:26.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:26.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:23:27.414 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-20T21:23:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:23:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:23:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:23:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:23:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:23:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:23:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:23:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:23:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:27.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:23:27.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:23:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:23:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:23:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:23:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:23:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:23:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:23:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:23:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:27.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:23:27.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:23:27.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:27.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:27.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:23:27.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:23:27.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:27.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:27.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:27.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:27.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:27.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:27.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:27.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:23:27.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:23:27.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:23:27.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:27.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:27.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:27.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:23:27.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:23:27.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:23:27.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:23:27.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:27.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:23:27.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:23:27.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:27.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:27.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:23:27.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:23:27.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:27.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:27.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:27.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:23:27.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:23:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:23:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:23:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:23:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:23:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:23:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:23:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:23:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:23:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:23:27.484 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:27.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:27.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:27.485 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:23:27.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:23:27.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:23:27.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:23:27.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:23:27.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:23:27.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:27.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ request_resync_image cluster1 mirror test image_id 2026-02-20T21:23:27.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:27.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:27.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:27.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id_var_name=image_id 2026-02-20T21:23:27.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:23:27.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:27.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:23:27.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:23:27.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:23:27.534 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:23:27.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval 'image_id='\''16faab1bc6b8'\''' 2026-02-20T21:23:27.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ image_id=16faab1bc6b8 2026-02-20T21:23:27.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval 'test -n "$image_id"' 2026-02-20T21:23:27.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -n 16faab1bc6b8 2026-02-20T21:23:27.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/test 2026-02-20T21:23:27.590 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Flagged image for resync from primary 2026-02-20T21:23:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemons cluster1 rbd mirror start mirror/test 2026-02-20T21:23:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_instance=cluster1 2026-02-20T21:23:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance=cluster1 2026-02-20T21:23:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loop_instance 2026-02-20T21:23:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:23:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:23:27.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:23:27.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:0 rbd mirror start mirror/test 2026-02-20T21:23:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:23:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:23:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:23:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:23:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:23:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:23:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:23:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:23:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:23:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:23:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:23:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:23:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:23:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:23:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:23:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:23:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:27.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:27.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:23:27.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:27.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:23:27.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:23:27.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:23:27.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:27.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:23:27.607 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:27.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:27.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:27.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start mirror/test 2026-02-20T21:23:27.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:23:27.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror start : start rbd mirror 2026-02-20T21:23:27.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:23:27.650 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:1 rbd mirror start mirror/test 2026-02-20T21:23:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:23:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:23:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:23:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:23:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:23:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:23:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:23:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:23:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:23:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:23:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:23:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:23:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:23:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:23:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:23:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:23:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:23:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:23:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:23:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:23:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:23:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:23:27.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:27.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:27.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:27.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror start mirror/test 2026-02-20T21:23:27.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:27.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror test deleted 16faab1bc6b8 2026-02-20T21:23:27.706 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:27.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:27.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:27.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=deleted 2026-02-20T21:23:27.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id=16faab1bc6b8 2026-02-20T21:23:27.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:23:27.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n 16faab1bc6b8 2026-02-20T21:23:27.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:23:27.707 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:23:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test deleted 16faab1bc6b8 2026-02-20T21:23:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=deleted 2026-02-20T21:23:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id=16faab1bc6b8 2026-02-20T21:23:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:23:27.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:23:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:23:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:23:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:23:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:23:27.809 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:23:27.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:23:27.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id= 2026-02-20T21:23:27.832 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:23:27.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test deleted = deleted 2026-02-20T21:23:27.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:27.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemons cluster1 rbd mirror start mirror/test 2026-02-20T21:23:27.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_instance=cluster1 2026-02-20T21:23:27.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:27.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:27.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance=cluster1 2026-02-20T21:23:27.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loop_instance 2026-02-20T21:23:27.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:27.833 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:23:27.834 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:23:27.834 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:23:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:0 rbd mirror start mirror/test 2026-02-20T21:23:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:23:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:23:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:23:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:23:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:23:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:23:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:23:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:23:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:23:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:27.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:23:27.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:23:27.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:23:27.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:23:27.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:27.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:27.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:23:27.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:23:27.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:23:27.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:27.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:27.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:23:27.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:27.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:23:27.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:23:27.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:23:27.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:27.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:23:27.845 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:27.846 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:27.846 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:27.846 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start mirror/test 2026-02-20T21:23:27.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:23:27.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror start : start rbd mirror 2026-02-20T21:23:27.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:23:27.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:27.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ admin_daemon cluster1:1 rbd mirror start mirror/test 2026-02-20T21:23:27.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:23:27.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:27.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:23:27.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:23:27.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:23:27.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:23:27.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:23:27.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:23:27.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:23:27.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:27.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:23:27.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:27.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:23:27.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:23:27.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:23:27.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:23:27.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:27.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:27.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:23:27.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:23:27.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:23:27.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:27.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:27.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:23:27.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:27.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:23:27.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:23:27.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:23:27.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:23:27.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:23:27.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:27.900 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:27.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:27.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror start mirror/test 2026-02-20T21:23:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror test present 2026-02-20T21:23:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=present 2026-02-20T21:23:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:23:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:23:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:23:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:23:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:23:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:23:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:23:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:23:27.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image_id=16fa5da42a48 2026-02-20T21:23:27.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:23:27.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:23:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test present 16fa5da42a48 2026-02-20T21:23:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:28.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:28.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=present 2026-02-20T21:23:28.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id=16fa5da42a48 2026-02-20T21:23:28.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:23:28.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:23:28.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:23:28.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:28.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:23:28.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:23:28.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:23:28.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:23:28.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id=16fa5da42a48 2026-02-20T21:23:28.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n 16fa5da42a48 2026-02-20T21:23:28.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -z 16fa5da42a48 -o 16fa5da42a48 = 16fa5da42a48 2026-02-20T21:23:28.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=present 2026-02-20T21:23:28.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test present = present 2026-02-20T21:23:28.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:28.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:23:28.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:28.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:28.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:28.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:23:28.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:28.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:28.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:28.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:23:28.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:23:28.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:28.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:23:29.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:23:29.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:29.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:29.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:29.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:23:29.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:23:29.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:23:29.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:23:29.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:23:29.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:23:29.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:29.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:29.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:23:29.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:23:29.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:29.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:23:29.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:23:29.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:23:29.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:29.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:23:29.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:23:29.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:29.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:29.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:23:29.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:23:29.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:29.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:29.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:29.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:23:29.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:29.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:29.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:29.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:23:29.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:29.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:23:29.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:23:29.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:29.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:29.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:23:29.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:23:29.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:29.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:29.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:29.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:23:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:23:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:23:29.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:29.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:29.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:29.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:23:29.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:23:29.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:23:29.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:23:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:23:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:23:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:23:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:23:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:23:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:23:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:23:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:23:29.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:23:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:23:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:23:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:23:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:29.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:29.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:29.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:23:29.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:23:29.169 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:23:29.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:29.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:29.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:29.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:23:29.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:23:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:23:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:23:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:23:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:23:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:23:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:23:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-20T21:23:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:23:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:23:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:23:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:23:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:23:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:23:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:23:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:23:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:23:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:23:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:23:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:23:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:23:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:23:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: d5138118-365e-4c80-9761-89b69f10c700 2026-02-20T21:23:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:23:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622549,"replay_state":"idle"} 2026-02-20T21:23:30.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:23:30.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:23:27 2026-02-20T21:23:30.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:23:30.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:23:30.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: down+unknown 2026-02-20T21:23:30.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: status not found 2026-02-20T21:23:30.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: ' 2026-02-20T21:23:30.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:23:30.259 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:23:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:23:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622549,"replay_state":"idle"} 2026-02-20T21:23:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:23:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:23:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:23:30.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:23:30.261 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:23:30.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:23:30.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:23:30.263 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:23:30.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:23:27' 2026-02-20T21:23:30.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:23:27' 2026-02-20T21:23:30.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:30.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:30.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:30.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:23:30.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:23:27' 2026-02-20T21:23:30.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:23:30.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:23:30.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:23:30.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING158775882mirror.1trial19920.2.0-677-gf7870454trueOK58845890mirror.2trial19920.2.0-677-gf7870454falseOK58915899mirror.3trial19920.2.0-677-gf7870454falseOK59065912mirror.0trial19920.2.0-677-gf7870454falseOKtestd5138118-365e-4c80-9761-89b69f10c700up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622549,"replay_state":"idle"}58775882mirror.1trial1992026-02-20 21:23:27cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8down+unknownstatus not found' 2026-02-20T21:23:30.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:23:30.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:23:30.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:23:27' 2026-02-20T21:23:30.299 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:23:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:23:30.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:23:30.301 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:23:30.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:23:30.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:30.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:30.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror mirror test 2026-02-20T21:23:30.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:23:30.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:23:30.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:30.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:23:30.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:23:30.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:30.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.export 2026-02-20T21:23:30.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.export 2026-02-20T21:23:30.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:23:30.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export 2026-02-20T21:23:30.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.export 2026-02-20T21:23:30.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.export 2026-02-20T21:23:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:23:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:23:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:23:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export 2026-02-20T21:23:30.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:23:30.401 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:23:30.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:23:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:23:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:23:30.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:30.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: request image resync while daemon is offline' 2026-02-20T21:23:30.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:23:30.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:23:30.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:23:30 TEST: request image resync while daemon is offline 2026-02-20T21:23:30.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:23:30 TEST: request image resync while daemon is offline 2026-02-20T21:23:30.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirrors cluster1 2026-02-20T21:23:30.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:30.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:23:30.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:23:30.581 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:30.581 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:0 '' 2026-02-20T21:23:30.581 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:23:30.581 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:23:30.581 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:23:30.581 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:23:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:0 2026-02-20T21:23:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:0 2026-02-20T21:23:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:23:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:23:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:23:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:23:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:23:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:23:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:23:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-20T21:23:30.594 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:23:30.595 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:23:30.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=81117 2026-02-20T21:23:30.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 81117 ']' 2026-02-20T21:23:30.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 81117 2026-02-20T21:23:30.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:23:30.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:23:30.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:23:30.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=81117 '$2 == pid {print; exit 1}' 2026-02-20T21:23:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:ubuntu 81117 0.2 0.0 0 0 ? Rs 21:22 0:00 [rbd-mirror] 2026-02-20T21:23:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:23:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:23:31.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:23:31.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=81117 '$2 == pid {print; exit 1}' 2026-02-20T21:23:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:23:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:23:31.622 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=81117 '$2 == pid {print; exit 1}' 2026-02-20T21:23:31.633 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:23:31.633 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:23:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:23:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:23:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:23:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:23:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:23:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:23:31.635 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:23:31.635 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:31.635 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:23:31.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:31.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-20T21:23:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:23:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:23:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:23:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:23:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:23:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:23:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:23:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:23:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:23:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:23:31.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:31.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:31.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:0 2026-02-20T21:23:31.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:23:31.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:31.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:23:31.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:23:31.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:31.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:31.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:31.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:23:31.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:31.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:31.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:31.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:23:31.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:31.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-20T21:23:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:23:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:23:31.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:31.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:1 '' 2026-02-20T21:23:31.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:23:31.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:23:31.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:23:31.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:23:31.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:1 2026-02-20T21:23:31.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:1 2026-02-20T21:23:31.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:23:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:23:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:23:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:23:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:23:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:23:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:23:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:23:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-20T21:23:31.686 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:23:31.686 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:23:31.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=81137 2026-02-20T21:23:31.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 81137 ']' 2026-02-20T21:23:31.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 81137 2026-02-20T21:23:31.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:23:31.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:23:31.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:23:31.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=81137 '$2 == pid {print; exit 1}' 2026-02-20T21:23:31.698 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:ubuntu 81137 0.7 0.0 745492 63888 ? Ssl 21:22 0:00 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:23:31.699 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:23:31.699 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:23:32.700 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:23:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=81137 '$2 == pid {print; exit 1}' 2026-02-20T21:23:32.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:23:32.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:23:32.713 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=81137 '$2 == pid {print; exit 1}' 2026-02-20T21:23:32.724 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:23:32.724 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:23:32.724 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:32.724 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:32.724 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:23:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:23:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:23:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:23:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:23:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:23:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:23:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:23:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:23:32.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:32.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:32.738 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-20T21:23:32.738 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:23:32.738 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:23:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:23:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:23:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:23:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:23:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:23:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:23:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:23:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:23:32.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:23:32.748 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:32.749 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:23:32.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:1 2026-02-20T21:23:32.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:23:32.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:32.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:23:32.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:23:32.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:32.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:32.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:32.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:23:32.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:32.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:32.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:32.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:23:32.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:23:32.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-20T21:23:32.762 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:23:32.762 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:23:32.764 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:32.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:2 '' 2026-02-20T21:23:32.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:23:32.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:23:32.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:23:32.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:23:32.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:2 2026-02-20T21:23:32.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:2 2026-02-20T21:23:32.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:32.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:23:32.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:23:32.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:23:32.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:32.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:32.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:23:32.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:23:32.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:23:32.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:23:32.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:23:32.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:23:32.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-20T21:23:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:23:32.776 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:23:32.777 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=81172 2026-02-20T21:23:32.777 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 81172 ']' 2026-02-20T21:23:32.777 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 81172 2026-02-20T21:23:32.777 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:23:32.777 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:23:32.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:23:32.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=81172 '$2 == pid {print; exit 1}' 2026-02-20T21:23:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:ubuntu 81172 0.1 0.0 305968 46784 ? Rsl 21:22 0:00 rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:23:32.789 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:23:32.789 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:23:33.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:23:33.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=81172 '$2 == pid {print; exit 1}' 2026-02-20T21:23:33.802 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:23:33.802 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:23:33.802 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=81172 '$2 == pid {print; exit 1}' 2026-02-20T21:23:33.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:23:33.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:23:33.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:33.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:33.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:23:33.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:23:33.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:23:33.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:33.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:33.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:23:33.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:23:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:23:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:23:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:23:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:23:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:23:33.825 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:23:33.826 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:23:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-20T21:23:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:23:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:23:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:23:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:23:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:23:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:23:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:23:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:23:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:23:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:23:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:23:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:23:33.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:23:33.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:23:33.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:2 2026-02-20T21:23:33.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:23:33.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:33.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:23:33.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:23:33.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:33.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:33.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:33.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:23:33.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:23:33.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:33.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:33.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:23:33.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:23:33.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-20T21:23:33.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:23:33.852 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:23:33.853 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:33.853 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:3 '' 2026-02-20T21:23:33.853 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:23:33.853 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:23:33.853 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:23:33.853 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:23:33.854 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:3 2026-02-20T21:23:33.854 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:3 2026-02-20T21:23:33.854 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:33.854 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:23:33.854 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:23:33.854 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:23:33.854 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:33.854 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:33.854 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:23:33.854 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:23:33.854 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:23:33.854 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:23:33.855 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:23:33.855 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:23:33.855 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-20T21:23:33.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:23:33.865 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:23:33.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=81227 2026-02-20T21:23:33.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 81227 ']' 2026-02-20T21:23:33.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 81227 2026-02-20T21:23:33.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:23:33.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:23:33.867 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:23:33.868 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=81227 '$2 == pid {print; exit 1}' 2026-02-20T21:23:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:ubuntu 81227 0.1 0.0 475016 47764 ? Rsl 21:22 0:00 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:23:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:23:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:23:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:23:34.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=81227 '$2 == pid {print; exit 1}' 2026-02-20T21:23:34.890 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:23:34.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:23:34.891 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=81227 '$2 == pid {print; exit 1}' 2026-02-20T21:23:34.901 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:23:34.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:23:34.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:34.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:34.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:23:34.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:23:34.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:23:34.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:34.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:34.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:23:34.902 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:23:34.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:23:34.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:23:34.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:23:34.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:23:34.903 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:23:34.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:23:34.913 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:23:34.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-20T21:23:34.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:23:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:23:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:23:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:23:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:23:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:23:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:23:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:23:34.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:23:34.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:23:34.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:23:34.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:23:34.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:23:34.928 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:23:34.930 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:3 2026-02-20T21:23:34.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:23:34.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:34.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:23:34.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:23:34.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:34.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:34.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:34.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:23:34.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:23:34.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:34.931 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:34.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:23:34.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:23:34.932 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-20T21:23:34.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:23:34.942 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:23:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ request_resync_image cluster1 mirror test image_id 2026-02-20T21:23:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id_var_name=image_id 2026-02-20T21:23:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:23:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:23:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:23:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:23:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:23:34.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval 'image_id='\''16fa5da42a48'\''' 2026-02-20T21:23:34.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ image_id=16fa5da42a48 2026-02-20T21:23:34.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval 'test -n "$image_id"' 2026-02-20T21:23:34.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -n 16fa5da42a48 2026-02-20T21:23:34.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/test 2026-02-20T21:23:34.998 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Flagged image for resync from primary 2026-02-20T21:23:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirrors cluster1 2026-02-20T21:23:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:35.001 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:23:35.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:35.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:0 2026-02-20T21:23:35.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:23:35.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:23:35.002 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:23:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:23:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:23:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:23:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:23:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:23:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:23:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:23:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:23:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:23:35.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:23:35.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:35.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:1 2026-02-20T21:23:35.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:23:35.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:23:35.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:23:35.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:23:35.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:23:35.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:23:35.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:23:35.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:23:35.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:23:35.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:23:35.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:35.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:23:35.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:23:35.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:23:35.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:23:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:2 2026-02-20T21:23:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:23:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:23:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:23:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:2 2026-02-20T21:23:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:23:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:23:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:23:35.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=2 2026-02-20T21:23:35.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:23:35.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:23:35.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:35.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=2 2026-02-20T21:23:35.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:23:35.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:23:35.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:23:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:3 2026-02-20T21:23:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:23:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:23:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:23:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:3 2026-02-20T21:23:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:23:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:23:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:23:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=3 2026-02-20T21:23:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:23:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:23:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=3 2026-02-20T21:23:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:23:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:23:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:23:35.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror test deleted 16fa5da42a48 2026-02-20T21:23:35.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:35.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:35.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:35.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=deleted 2026-02-20T21:23:35.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id=16fa5da42a48 2026-02-20T21:23:35.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:23:35.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n 16fa5da42a48 2026-02-20T21:23:35.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:23:35.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:23:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test deleted 16fa5da42a48 2026-02-20T21:23:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=deleted 2026-02-20T21:23:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id=16fa5da42a48 2026-02-20T21:23:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:23:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:23:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:23:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:23:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:23:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:23:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:23:35.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id=16fa5da42a48 2026-02-20T21:23:35.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n 16fa5da42a48 2026-02-20T21:23:35.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -z 16fa5da42a48 -o 16fa5da42a48 = 16fa5da42a48 2026-02-20T21:23:35.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=present 2026-02-20T21:23:35.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test deleted = present 2026-02-20T21:23:35.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:23:35.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:23:36.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test deleted 16fa5da42a48 2026-02-20T21:23:36.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:36.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:36.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:36.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=deleted 2026-02-20T21:23:36.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id=16fa5da42a48 2026-02-20T21:23:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:23:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:23:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:23:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:23:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:23:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:23:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:23:36.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id=16fa5da42a48 2026-02-20T21:23:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n 16fa5da42a48 2026-02-20T21:23:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -z 16fa5da42a48 -o 16fa5da42a48 = 16fa5da42a48 2026-02-20T21:23:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=present 2026-02-20T21:23:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test deleted = present 2026-02-20T21:23:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:23:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:23:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test deleted 16fa5da42a48 2026-02-20T21:23:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:38.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=deleted 2026-02-20T21:23:38.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id=16fa5da42a48 2026-02-20T21:23:38.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:23:38.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:23:38.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:23:38.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:38.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:23:38.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:23:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:23:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:23:38.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id=16fa5da42a48 2026-02-20T21:23:38.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n 16fa5da42a48 2026-02-20T21:23:38.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -z 16fa5da42a48 -o 16fa5da42a48 = 16fa5da42a48 2026-02-20T21:23:38.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=present 2026-02-20T21:23:38.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test deleted = present 2026-02-20T21:23:38.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:23:38.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:23:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test deleted 16fa5da42a48 2026-02-20T21:23:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=deleted 2026-02-20T21:23:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id=16fa5da42a48 2026-02-20T21:23:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:23:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:23:42.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:23:42.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:42.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:23:42.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:23:42.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:23:42.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:23:42.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id=17eaa326dc44 2026-02-20T21:23:42.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n 17eaa326dc44 2026-02-20T21:23:42.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -z 16fa5da42a48 -o 16fa5da42a48 = 17eaa326dc44 2026-02-20T21:23:42.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test deleted = deleted 2026-02-20T21:23:42.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:42.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_present cluster1 mirror test present 2026-02-20T21:23:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=present 2026-02-20T21:23:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id= 2026-02-20T21:23:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:23:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:23:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:23:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:23:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:23:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:23:42.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:23:42.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image_id=17eaa326dc44 2026-02-20T21:23:42.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:23:42.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:23:42.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_present cluster1 mirror test present 17eaa326dc44 2026-02-20T21:23:42.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:42.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:42.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:42.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=present 2026-02-20T21:23:42.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id=17eaa326dc44 2026-02-20T21:23:42.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=deleted 2026-02-20T21:23:42.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_image_id 2026-02-20T21:23:42.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:23:42.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:42.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:23:42.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test 2026-02-20T21:23:42.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:23:42.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:23:42.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_image_id=17eaa326dc44 2026-02-20T21:23:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n 17eaa326dc44 2026-02-20T21:23:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -z 17eaa326dc44 -o 17eaa326dc44 = 17eaa326dc44 2026-02-20T21:23:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=present 2026-02-20T21:23:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test present = present 2026-02-20T21:23:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:23:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:23:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:23:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:23:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:23:43.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:23:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:23:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:23:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:23:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:23:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:23:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:23:43.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:23:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:23:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:23:43.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:23:43.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:23:43.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:43.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:23:43.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:23:43.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:43.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:43.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:23:43.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:23:43.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:43.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:43.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:23:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:23:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:23:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:23:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:23:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:23:43.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:43.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:43.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:43.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:43.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:43.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:43.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:43.475 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:23:43.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:23:43.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:23:43.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:43.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:43.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:43.481 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:23:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:23:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:23:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:23:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:23:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:23:43.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:23:43.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:23:43.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:43.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-20T21:23:43.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:43.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:43.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:43.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:23:43.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:23:43.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:43.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:43.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:43.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:23:43.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:23:43.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:23:43.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:43.533 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:23:44.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:23:44.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:44.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:44.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:44.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:23:44.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:23:44.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:23:44.531 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:23:44.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:23:44.532 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:23:44.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test: 2026-02-20T21:23:44.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: d5138118-365e-4c80-9761-89b69f10c700 2026-02-20T21:23:44.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:23:44.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622549,"replay_state":"idle"} 2026-02-20T21:23:44.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:23:44.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:23:40 2026-02-20T21:23:44.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:23:44.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:23:44.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: down+unknown 2026-02-20T21:23:44.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: status not found 2026-02-20T21:23:44.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: ' 2026-02-20T21:23:44.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:23:44.567 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:23:44.567 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:23:44.568 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622549,"replay_state":"idle"} 2026-02-20T21:23:44.568 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:23:44.568 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:23:44.568 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:23:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:23:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:23:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:23:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:23:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:23:44.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:23:40' 2026-02-20T21:23:44.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:23:40' 2026-02-20T21:23:44.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:44.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:44.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:44.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:23:44.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:23:40' 2026-02-20T21:23:44.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:23:44.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:23:44.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:23:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING161176122mirror.0trial19920.2.0-677-gf7870454trueOK61246130mirror.1trial19920.2.0-677-gf7870454falseOK61316138mirror.2trial19920.2.0-677-gf7870454falseOK61416147mirror.3trial19920.2.0-677-gf7870454falseOKtestd5138118-365e-4c80-9761-89b69f10c700up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622549,"replay_state":"idle"}61176122mirror.0trial1992026-02-20 21:23:40cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8down+unknownstatus not found' 2026-02-20T21:23:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:23:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:23:44.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:23:40' 2026-02-20T21:23:44.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:23:44.607 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:23:44.607 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:23:44.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:23:44.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:23:44.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:44.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:44.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror mirror test 2026-02-20T21:23:44.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:23:44.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:23:44.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:44.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:23:44.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:23:44.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:44.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test.export 2026-02-20T21:23:44.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test.export 2026-02-20T21:23:44.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:23:44.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export 2026-02-20T21:23:44.611 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test.export 2026-02-20T21:23:44.611 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test.export 2026-02-20T21:23:44.611 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:23:44.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:23:44.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:23:44.613 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export 2026-02-20T21:23:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:23:44.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:23:44.790 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:23:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:23:44.846 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test.export 2026-02-20T21:23:44.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:44.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster2 mirror test 2026-02-20T21:23:44.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:44.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:44.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:44.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:23:44.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:23:44.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster2 mirror test 2026-02-20T21:23:44.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:44.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:44.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test 2026-02-20T21:23:44.880 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 snap purge mirror/test 2026-02-20T21:23:44.910 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rm mirror/test 2026-02-20T21:23:45.007 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:23:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:23:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: split-brain' 2026-02-20T21:23:45.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:23:45.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:23:45.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:23:45 TEST: split-brain 2026-02-20T21:23:45.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:23:45 TEST: split-brain 2026-02-20T21:23:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ image=split-brain 2026-02-20T21:23:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image_and_enable_mirror cluster2 mirror split-brain snapshot 2026-02-20T21:23:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:23:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:23:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:23:45.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image cluster2 mirror split-brain 2026-02-20T21:23:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:23:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:23:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=128 2026-02-20T21:23:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:23:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,object-map,fast-diff mirror/split-brain 2026-02-20T21:23:45.041 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:23:45.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:45.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:23:45.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:45.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror split-brain snapshot 2026-02-20T21:23:45.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:23:45.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:45.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:23:45.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:23:45.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/split-brain snapshot 2026-02-20T21:23:45.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:23:45.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/split-brain 2026-02-20T21:23:46.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'split-brain': 2026-02-20T21:23:46.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 128 MiB in 32 objects 2026-02-20T21:23:46.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:23:46.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:23:46.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 18a8a531465e 2026-02-20T21:23:46.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.18a8a531465e 2026-02-20T21:23:46.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:23:46.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:23:46.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:23:46.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:23:46.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:23:45 2026 2026-02-20T21:23:46.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:23:45 2026 2026-02-20T21:23:46.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:23:45 2026 2026-02-20T21:23:46.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:23:46.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:23:46.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 7ff13149-8438-4f4f-8886-adba99752fd4 2026-02-20T21:23:46.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:23:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror split-brain 2026-02-20T21:23:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:23:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:23:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror split-brain up+replaying 2026-02-20T21:23:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:46.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:46.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:23:46.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:23:46.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:23:46.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:23:46.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:46.026 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:23:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying '' '' 2026-02-20T21:23:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:23:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:23:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:23:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:23:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:23:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:23:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-20T21:23:47.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image split-brain: (2) No such file or directory 2026-02-20T21:23:47.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status= 2026-02-20T21:23:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:23:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 1 2026-02-20T21:23:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:23:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying '' '' 2026-02-20T21:23:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:23:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:23:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:23:49.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:23:49.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:23:49.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:23:49.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-20T21:23:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='split-brain: 2026-02-20T21:23:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 7ff13149-8438-4f4f-8886-adba99752fd4 2026-02-20T21:23:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:23:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622625,"replay_state":"idle"} 2026-02-20T21:23:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:23:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:23:47 2026-02-20T21:23:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:23:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:23:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: down+unknown 2026-02-20T21:23:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: status not found 2026-02-20T21:23:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: ' 2026-02-20T21:23:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:23:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:23:49.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:23:49.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622625,"replay_state":"idle"} 2026-02-20T21:23:49.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:23:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:23:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:23:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:23:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:23:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:23:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:23:49.085 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:23:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:23:47' 2026-02-20T21:23:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror split-brain up+replaying '2026-02-20 21:23:47' 2026-02-20T21:23:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:23:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:23:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:23:47' 2026-02-20T21:23:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:23:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:23:49.088 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:23:49.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING161176122mirror.0trial19920.2.0-677-gf7870454trueOK61246130mirror.1trial19920.2.0-677-gf7870454falseOK61316138mirror.2trial19920.2.0-677-gf7870454falseOK61416147mirror.3trial19920.2.0-677-gf7870454falseOKsplit-brain7ff13149-8438-4f4f-8886-adba99752fd4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622625,"replay_state":"idle"}61176122mirror.0trial1992026-02-20 21:23:47cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8down+unknownstatus not found' 2026-02-20T21:23:49.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:23:49.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/last_update' 2026-02-20T21:23:49.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:23:47' 2026-02-20T21:23:49.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/state' 2026-02-20T21:23:49.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:23:49.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:23:49.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:23:49.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:23:49.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:49.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:49.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror split-brain --force 2026-02-20T21:23:49.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:49.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:49.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:23:49.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force=--force 2026-02-20T21:23:49.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/split-brain --force 2026-02-20T21:23:57.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:23:57.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror split-brain 2026-02-20T21:23:57.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:57.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:57.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:23:57.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror split-brain stopped 2026-02-20T21:23:57.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:57.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:57.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:23:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:23:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:23:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:23:58.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror split-brain stopped 2026-02-20T21:23:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:23:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:23:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:23:58.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:23:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/split-brain 2026-02-20T21:23:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:23:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:23:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:23:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:23:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:23:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:23:58.971 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:23:58.972 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:23:58.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:23:58.974 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/split-brain 2026-02-20T21:23:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:23:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:23:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:23:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:23:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:23:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:23:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:23:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:23:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:23:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:23:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:23:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:23:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:23:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:23:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:23:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:23:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:23:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:23:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:23:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:23:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:23:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:23:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:23:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:23:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:23:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:23:58.977 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:23:58.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:58.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:58.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:23:58.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/split-brain 2026-02-20T21:23:59.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:23:59.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:23:59.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:23:59.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:23:59.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:23:59.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:23:59.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror split-brain up+stopped 2026-02-20T21:23:59.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:23:59.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:23:59.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:23:59.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:23:59.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:23:59.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:23:59.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:23:59.036 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:24:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+stopped '' '' 2026-02-20T21:24:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:24:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:24:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:24:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:24:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:24:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:24:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-20T21:24:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='split-brain: 2026-02-20T21:24:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 7ff13149-8438-4f4f-8886-adba99752fd4 2026-02-20T21:24:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:24:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:24:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:24:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:24:00 2026-02-20T21:24:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:24:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:24:00.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: down+unknown 2026-02-20T21:24:00.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: status not found 2026-02-20T21:24:00.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20T21:24:00.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:24:00.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 208 .mirror.primary.7ff13149-8438-4f4f-8886-adba99752fd4.7597abf0-2cc6-49a3-9104-52ad6598c37e (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:24:00.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:24:00.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:24:00.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:24:00.071 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:24:00.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:24:00.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:24:00.072 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:24:00.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:24:00.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:24:00.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:24:00.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:24:00.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:24:00.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:24:00' 2026-02-20T21:24:00.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror split-brain up+stopped '2026-02-20 21:24:00' 2026-02-20T21:24:00.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:00.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:00.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:24:00.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:24:00.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:24:00' 2026-02-20T21:24:00.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:24:00.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:24:00.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:24:00.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING161176122mirror.0trial19920.2.0-677-gf7870454trueOK61246130mirror.1trial19920.2.0-677-gf7870454falseOK61316138mirror.2trial19920.2.0-677-gf7870454falseOK61416147mirror.3trial19920.2.0-677-gf7870454falseOKsplit-brain7ff13149-8438-4f4f-8886-adba99752fd4up+stoppedlocal image is primary61176122mirror.0trial1992026-02-20 21:24:00cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8down+unknownstatus not found' 2026-02-20T21:24:00.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:24:00.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/last_update' 2026-02-20T21:24:00.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:24:00' 2026-02-20T21:24:00.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/state' 2026-02-20T21:24:00.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:24:00.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:24:00.110 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:24:00.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:24:00.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:24:00.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:24:00.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster1 mirror split-brain 10 2026-02-20T21:24:00.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:00.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:00.112 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:24:00.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=10 2026-02-20T21:24:00.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:24:00.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:24:00.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:24:00.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 bench mirror/split-brain --io-type write --io-size 4096 --io-threads 1 --io-total 40960 --io-pattern rand 2026-02-20T21:24:00.141 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 40960 pattern random 2026-02-20T21:24:00.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:24:00.147 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 10 ops/sec: 1666.69 bytes/sec: 6.5 MiB/s 2026-02-20T21:24:00.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster1 mirror split-brain 2026-02-20T21:24:00.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:00.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:00.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:24:00.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/split-brain 2026-02-20T21:24:00.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:24:00.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror split-brain up+error split-brain 2026-02-20T21:24:00.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:00.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:00.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:24:00.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+error 2026-02-20T21:24:00.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern=split-brain 2026-02-20T21:24:00.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:24:00.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:24:00.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:24:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+error split-brain '' 2026-02-20T21:24:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:24:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+error 2026-02-20T21:24:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern=split-brain 2026-02-20T21:24:01.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:24:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:24:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:24:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-20T21:24:02.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='split-brain: 2026-02-20T21:24:02.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 7ff13149-8438-4f4f-8886-adba99752fd4 2026-02-20T21:24:02.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:24:02.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:24:02.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:24:02.021 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:24:00 2026-02-20T21:24:02.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:24:02.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:24:02.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: down+unknown 2026-02-20T21:24:02.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: status not found 2026-02-20T21:24:02.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: ' 2026-02-20T21:24:02.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+error' 2026-02-20T21:24:02.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 1 2026-02-20T21:24:02.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:24:02.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:24:04.023 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+error split-brain '' 2026-02-20T21:24:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:24:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+error 2026-02-20T21:24:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern=split-brain 2026-02-20T21:24:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:24:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:24:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:24:04.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-20T21:24:04.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='split-brain: 2026-02-20T21:24:04.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 7ff13149-8438-4f4f-8886-adba99752fd4 2026-02-20T21:24:04.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:24:04.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:24:04.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:24:04.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:24:00 2026-02-20T21:24:04.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:24:04.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:24:04.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: down+unknown 2026-02-20T21:24:04.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: status not found 2026-02-20T21:24:04.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: ' 2026-02-20T21:24:04.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+error' 2026-02-20T21:24:04.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 1 2026-02-20T21:24:04.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:24:04.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:24:08.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+error split-brain '' 2026-02-20T21:24:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:24:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+error 2026-02-20T21:24:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern=split-brain 2026-02-20T21:24:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:24:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:24:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:24:08.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-20T21:24:08.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='split-brain: 2026-02-20T21:24:08.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 7ff13149-8438-4f4f-8886-adba99752fd4 2026-02-20T21:24:08.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+error 2026-02-20T21:24:08.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: split-brain 2026-02-20T21:24:08.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:24:08.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:24:05 2026-02-20T21:24:08.091 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:24:08.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:24:08.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: down+unknown 2026-02-20T21:24:08.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: status not found 2026-02-20T21:24:08.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: ' 2026-02-20T21:24:08.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+error' 2026-02-20T21:24:08.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+error 2026-02-20T21:24:08.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*split-brain' 2026-02-20T21:24:08.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: split-brain 2026-02-20T21:24:08.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:24:08.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+error 2026-02-20T21:24:08.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:24:08.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+error 2026-02-20T21:24:08.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:24:08.095 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:24:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:24:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:24:08.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:24:05' 2026-02-20T21:24:08.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror split-brain up+error '2026-02-20 21:24:05' 2026-02-20T21:24:08.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:08.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:08.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:24:08.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+error 2026-02-20T21:24:08.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:24:05' 2026-02-20T21:24:08.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:24:08.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:24:08.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:24:08.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING161176122mirror.0trial19920.2.0-677-gf7870454trueOK61246130mirror.1trial19920.2.0-677-gf7870454falseOK61316138mirror.2trial19920.2.0-677-gf7870454falseOK61416147mirror.3trial19920.2.0-677-gf7870454falseOKsplit-brain7ff13149-8438-4f4f-8886-adba99752fd4up+errorsplit-brain61176122mirror.0trial1992026-02-20 21:24:05cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8down+unknownstatus not found' 2026-02-20T21:24:08.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:24:08.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/last_update' 2026-02-20T21:24:08.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:24:05' 2026-02-20T21:24:08.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/state' 2026-02-20T21:24:08.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+error 2026-02-20T21:24:08.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+error 2026-02-20T21:24:08.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+error 2026-02-20T21:24:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+error 2026-02-20T21:24:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:24:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:24:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ request_resync_image cluster1 mirror split-brain image_id 2026-02-20T21:24:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:24:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image_id_var_name=image_id 2026-02-20T21:24:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_image_id cluster1 mirror split-brain 2026-02-20T21:24:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:24:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:24:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=split-brain 2026-02-20T21:24:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster=cluster1 info mirror/split-brain 2026-02-20T21:24:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:24:08.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval 'image_id='\''17ea3f4efd88'\''' 2026-02-20T21:24:08.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ image_id=17ea3f4efd88 2026-02-20T21:24:08.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval 'test -n "$image_id"' 2026-02-20T21:24:08.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -n 17ea3f4efd88 2026-02-20T21:24:08.161 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/split-brain 2026-02-20T21:24:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Flagged image for resync from primary 2026-02-20T21:24:08.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror split-brain 2026-02-20T21:24:08.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:08.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:08.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:24:08.191 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:24:08.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror split-brain up+replaying 2026-02-20T21:24:08.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:08.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:08.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:24:08.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:24:08.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:24:08.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:24:08.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:24:08.192 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:24:09.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying '' '' 2026-02-20T21:24:09.193 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:24:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:24:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:24:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:24:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:24:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:24:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-20T21:24:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='split-brain: 2026-02-20T21:24:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 7ff13149-8438-4f4f-8886-adba99752fd4 2026-02-20T21:24:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+error 2026-02-20T21:24:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: split-brain 2026-02-20T21:24:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:24:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:24:05 2026-02-20T21:24:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:24:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:24:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: down+unknown 2026-02-20T21:24:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: status not found 2026-02-20T21:24:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: ' 2026-02-20T21:24:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:24:09.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 1 2026-02-20T21:24:09.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:24:09.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:24:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying '' '' 2026-02-20T21:24:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:11.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:24:11.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:24:11.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:24:11.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:24:11.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:24:11.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:24:11.226 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-20T21:24:11.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: error opening image split-brain: (2) No such file or directory 2026-02-20T21:24:11.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status= 2026-02-20T21:24:11.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:24:11.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 1 2026-02-20T21:24:11.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:24:11.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:24:15.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying '' '' 2026-02-20T21:24:15.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:15.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:15.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:24:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:24:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:24:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:24:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:24:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:24:15.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-20T21:24:15.280 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd: mirroring not enabled on the image 2026-02-20T21:24:15.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status= 2026-02-20T21:24:15.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:24:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 1 2026-02-20T21:24:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:24:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 8 2026-02-20T21:24:23.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying '' '' 2026-02-20T21:24:23.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:23.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:23.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:24:23.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:24:23.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:24:23.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:24:23.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:24:23.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:24:23.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-20T21:24:23.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='split-brain: 2026-02-20T21:24:23.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 7ff13149-8438-4f4f-8886-adba99752fd4 2026-02-20T21:24:23.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:24:23.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622625,"replay_state":"idle"} 2026-02-20T21:24:23.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:24:23.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:24:15 2026-02-20T21:24:23.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:24:23.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:24:23.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: down+unknown 2026-02-20T21:24:23.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: status not found 2026-02-20T21:24:23.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: ' 2026-02-20T21:24:23.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:24:23.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:24:23.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:24:23.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622625,"replay_state":"idle"} 2026-02-20T21:24:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:24:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:24:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:24:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:24:23.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:24:23.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:24:23.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:24:23.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:24:23.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:24:15' 2026-02-20T21:24:23.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror split-brain up+replaying '2026-02-20 21:24:15' 2026-02-20T21:24:23.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:23.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:23.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:24:23.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:24:23.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:24:15' 2026-02-20T21:24:23.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:24:23.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:24:23.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:24:23.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING161176122mirror.0trial19920.2.0-677-gf7870454trueOK61246130mirror.1trial19920.2.0-677-gf7870454falseOK61316138mirror.2trial19920.2.0-677-gf7870454falseOK61416147mirror.3trial19920.2.0-677-gf7870454falseOKsplit-brain7ff13149-8438-4f4f-8886-adba99752fd4up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622625,"replay_state":"idle"}61176122mirror.0trial1992026-02-20 21:24:15cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8down+unknownstatus not found' 2026-02-20T21:24:23.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:24:23.356 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/last_update' 2026-02-20T21:24:23.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:24:15' 2026-02-20T21:24:23.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/state' 2026-02-20T21:24:23.359 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:24:23.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:24:23.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:24:23.360 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:24:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:24:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:24:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster2 mirror split-brain 2026-02-20T21:24:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:24:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:24:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:24:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster2 mirror split-brain 2026-02-20T21:24:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=split-brain 2026-02-20T21:24:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 snap purge mirror/split-brain 2026-02-20T21:24:23.389 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rm mirror/split-brain 2026-02-20T21:24:24.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:24:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:24:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: check if removed images'\'' OMAP are removed' 2026-02-20T21:24:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:24:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:24:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:24:24 TEST: check if removed 'images'\''' OMAP are removed 2026-02-20T21:24:24.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:24:24 TEST: check if removed images' OMAP are removed 2026-02-20T21:24:24.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirrors cluster2 2026-02-20T21:24:24.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:24.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:24:24.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:24:24.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster2:0 2026-02-20T21:24:24.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2:0 2026-02-20T21:24:24.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:24:24.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:24:24.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster2:0 2026-02-20T21:24:24.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:24:24.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:24:24.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster2 2026-02-20T21:24:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:24:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-20T21:24:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster2 2026-02-20T21:24:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:24:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:24:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:24:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:24:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster2 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:24:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:24:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster2:1 2026-02-20T21:24:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2:1 2026-02-20T21:24:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:24:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:24:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster2:1 2026-02-20T21:24:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:24:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:24:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster2 2026-02-20T21:24:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:24:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-20T21:24:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster2 2026-02-20T21:24:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:24:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:24:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:24:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:24:24.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster2 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:24:24.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:24:24.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster2:2 2026-02-20T21:24:24.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2:2 2026-02-20T21:24:24.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:24:24.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-20T21:24:24.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster2:2 2026-02-20T21:24:24.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:24:24.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:24:24.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster2 2026-02-20T21:24:24.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=2 2026-02-20T21:24:24.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-20T21:24:24.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster2 2026-02-20T21:24:24.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:24:24.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=2 2026-02-20T21:24:24.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:24:24.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:24:24.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster2 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:24:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:24:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster2:3 2026-02-20T21:24:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2:3 2026-02-20T21:24:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:24:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-20T21:24:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster2:3 2026-02-20T21:24:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:24:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:24:24.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster2 2026-02-20T21:24:24.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=3 2026-02-20T21:24:24.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-20T21:24:24.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster2 2026-02-20T21:24:24.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:24:24.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=3 2026-02-20T21:24:24.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:24:24.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:24:24.074 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster2 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:24:24.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_in_omap cluster1 mirror 2026-02-20T21:24:24.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:24.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:24.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring status_global 2026-02-20T21:24:24.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:24.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:24.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local obj_name=rbd_mirroring 2026-02-20T21:24:24.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local filter=status_global 2026-02-20T21:24:24.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T21:24:24.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:24:24.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set +e 2026-02-20T21:24:24.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring status_global 2026-02-20T21:24:24.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:24:24.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:24:24.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:24:24.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local filter=status_global 2026-02-20T21:24:24.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-20T21:24:24.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:24:24.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -c status_global 2026-02-20T21:24:24.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:24:24.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:24:24.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-20T21:24:24.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 0 = 0 2026-02-20T21:24:24.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ error_code=0 2026-02-20T21:24:24.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set -e 2026-02-20T21:24:24.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T21:24:24.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:24:24.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring image_ 2026-02-20T21:24:24.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:24.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:24.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local obj_name=rbd_mirroring 2026-02-20T21:24:24.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local filter=image_ 2026-02-20T21:24:24.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T21:24:24.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:24:24.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set +e 2026-02-20T21:24:24.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring image_ 2026-02-20T21:24:24.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:24:24.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:24:24.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:24:24.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local filter=image_ 2026-02-20T21:24:24.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-20T21:24:24.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -c image_ 2026-02-20T21:24:24.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:24:24.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:24:24.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:24:24.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-20T21:24:24.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 0 = 0 2026-02-20T21:24:24.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ error_code=0 2026-02-20T21:24:24.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set -e 2026-02-20T21:24:24.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T21:24:24.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:24:24.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirror_leader image_map 2026-02-20T21:24:24.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:24.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:24.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local obj_name=rbd_mirror_leader 2026-02-20T21:24:24.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local filter=image_map 2026-02-20T21:24:24.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T21:24:24.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:24:24.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set +e 2026-02-20T21:24:24.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-20T21:24:24.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:24:24.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:24:24.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T21:24:24.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local filter=image_map 2026-02-20T21:24:24.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-20T21:24:24.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:24:24.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -c image_map 2026-02-20T21:24:24.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:24:24.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T21:24:24.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-20T21:24:24.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 1 = 0 2026-02-20T21:24:24.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ error_code=1 2026-02-20T21:24:24.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set -e 2026-02-20T21:24:24.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 -eq 0 ']' 2026-02-20T21:24:24.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T21:24:24.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:24:25.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set +e 2026-02-20T21:24:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-20T21:24:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:24:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:24:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T21:24:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local filter=image_map 2026-02-20T21:24:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-20T21:24:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:24:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:24:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T21:24:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-20T21:24:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -c image_map 2026-02-20T21:24:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 1 = 0 2026-02-20T21:24:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ error_code=1 2026-02-20T21:24:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set -e 2026-02-20T21:24:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 -eq 0 ']' 2026-02-20T21:24:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T21:24:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:24:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set +e 2026-02-20T21:24:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-20T21:24:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:24:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:24:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T21:24:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local filter=image_map 2026-02-20T21:24:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-20T21:24:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:24:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:24:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T21:24:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-20T21:24:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -c image_map 2026-02-20T21:24:27.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 0 = 0 2026-02-20T21:24:27.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ error_code=0 2026-02-20T21:24:27.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set -e 2026-02-20T21:24:27.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T21:24:27.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:24:27.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_in_omap cluster2 mirror 2026-02-20T21:24:27.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:27.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:27.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring status_global 2026-02-20T21:24:27.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:27.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local obj_name=rbd_mirroring 2026-02-20T21:24:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local filter=status_global 2026-02-20T21:24:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T21:24:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:24:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set +e 2026-02-20T21:24:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring status_global 2026-02-20T21:24:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:24:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:24:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:24:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local filter=status_global 2026-02-20T21:24:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-20T21:24:27.228 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:24:27.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:24:27.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:24:27.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-20T21:24:27.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -c status_global 2026-02-20T21:24:27.248 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 0 = 0 2026-02-20T21:24:27.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ error_code=0 2026-02-20T21:24:27.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set -e 2026-02-20T21:24:27.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T21:24:27.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:24:27.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring image_ 2026-02-20T21:24:27.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:27.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:27.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local obj_name=rbd_mirroring 2026-02-20T21:24:27.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local filter=image_ 2026-02-20T21:24:27.249 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T21:24:27.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:24:27.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set +e 2026-02-20T21:24:27.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring image_ 2026-02-20T21:24:27.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:24:27.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:24:27.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:24:27.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local filter=image_ 2026-02-20T21:24:27.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-20T21:24:27.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:24:27.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:24:27.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:24:27.250 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-20T21:24:27.251 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -c image_ 2026-02-20T21:24:27.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 0 = 0 2026-02-20T21:24:27.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ error_code=0 2026-02-20T21:24:27.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set -e 2026-02-20T21:24:27.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T21:24:27.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:24:27.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirror_leader image_map 2026-02-20T21:24:27.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:27.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:27.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local obj_name=rbd_mirror_leader 2026-02-20T21:24:27.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local filter=image_map 2026-02-20T21:24:27.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T21:24:27.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:24:27.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set +e 2026-02-20T21:24:27.473 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirror_leader image_map 2026-02-20T21:24:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:24:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:24:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T21:24:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local filter=image_map 2026-02-20T21:24:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ list_omap_keys cluster2 mirror rbd_mirror_leader 2026-02-20T21:24:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:24:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:24:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T21:24:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -c image_map 2026-02-20T21:24:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirror_leader 2026-02-20T21:24:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 0 = 0 2026-02-20T21:24:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ error_code=0 2026-02-20T21:24:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set -e 2026-02-20T21:24:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T21:24:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:24:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -z '' ']' 2026-02-20T21:24:27.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: no blocklists' 2026-02-20T21:24:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:24:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:24:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:24:27 TEST: no blocklists 2026-02-20T21:24:27.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:24:27 TEST: no blocklists 2026-02-20T21:24:27.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS='--id admin' 2026-02-20T21:24:27.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --cluster cluster1 osd blocklist ls 2026-02-20T21:24:27.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -q 'listed 0 entries' 2026-02-20T21:24:27.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS='--id admin' 2026-02-20T21:24:27.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --cluster cluster2 osd blocklist ls 2026-02-20T21:24:27.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -q 'listed 0 entries' 2026-02-20T21:24:27.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:24:27.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -z '' ']' 2026-02-20T21:24:27.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: test partially synced demote snapshot sync after daemon restart' 2026-02-20T21:24:27.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:24:27.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:24:27.961 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:24:27 TEST: test partially synced demote snapshot sync after daemon restart 2026-02-20T21:24:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:24:27 TEST: test partially synced demote snapshot sync after daemon restart 2026-02-20T21:24:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image=test_demote_image 2026-02-20T21:24:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image_and_enable_mirror cluster2 mirror test_demote_image snapshot 10G 2026-02-20T21:24:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:24:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:24:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:24:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:24:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:24:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:24:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image cluster2 mirror test_demote_image 10G 2026-02-20T21:24:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:24:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:24:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:24:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=128 2026-02-20T21:24:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 10G ']' 2026-02-20T21:24:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=10G 2026-02-20T21:24:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:24:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/test_demote_image 2026-02-20T21:24:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:24:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:24:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:24:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror test_demote_image snapshot 2026-02-20T21:24:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:24:27.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_demote_image snapshot 2026-02-20T21:24:29.000 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:24:29.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/test_demote_image 2026-02-20T21:24:29.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test_demote_image': 2026-02-20T21:24:29.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 10 GiB in 2560 objects 2026-02-20T21:24:29.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:24:29.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:24:29.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 18ee6e310c1d 2026-02-20T21:24:29.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.18ee6e310c1d 2026-02-20T21:24:29.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:24:29.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:24:29.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:24:29.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:24:29.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:24:27 2026 2026-02-20T21:24:29.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:24:27 2026 2026-02-20T21:24:29.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:24:27 2026 2026-02-20T21:24:29.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:24:29.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:24:29.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 725c7e75-bf09-4751-a921-33bb6f9b4e5f 2026-02-20T21:24:29.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:24:29.031 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test_demote_image 100 2026-02-20T21:24:29.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:29.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:29.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:29.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:24:29.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:24:29.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:24:29.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:24:29.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test_demote_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:24:29.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:24:29.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:24:29.111 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 1960.81 bytes/sec: 7.7 MiB/s 2026-02-20T21:24:29.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_demote_image 2026-02-20T21:24:29.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:29.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:29.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:29.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test_demote_image stopped 2026-02-20T21:24:29.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:29.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:29.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:29.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:24:29.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:24:29.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:24:29.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:24:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test_demote_image stopped 2026-02-20T21:24:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:24:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:24:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:24:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-02-20T21:24:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:24:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:24:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:24:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:24:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:24:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:24:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:24:30.121 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:24:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:24:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:24:30.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:24:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-20T21:24:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:24:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:24:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:24:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:24:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:24:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:24:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:24:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:24:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:24:30.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:24:30.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:24:30.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:24:30.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:24:30.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:24:30.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:24:30.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:24:30.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:24:30.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:24:30.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:24:30.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:24:30.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:24:30.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:24:30.126 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:24:30.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:24:30.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:24:30.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:24:30.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:24:30.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:24:30.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:24:30.127 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:24:30.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:24:30.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:24:30.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:24:30.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-20T21:24:30.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-20T21:24:30.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:24:30.176 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:24:30.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:24:30.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_demote_image 2026-02-20T21:24:30.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:24:30.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:24:30.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:24:30.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:24:30.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:24:30.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:24:30.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:24:30.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:24:30.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:24:30.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:24:30.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:24:30.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:24:30.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:24:30.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:24:30.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:24:30.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:24:30.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:24:30.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:24:30.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:24:30.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:24:30.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:24:30.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:24:30.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:24:30.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:24:30.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:24:30.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:24:30.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:24:30.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:24:30.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:24:30.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:24:30.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:24:30.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:24:30.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:24:30.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-02-20T21:24:30.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-20T21:24:30.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:24:30.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:24:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:24:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_demote_image 2026-02-20T21:24:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:2 2026-02-20T21:24:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:24:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:24:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-20T21:24:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:2 2026-02-20T21:24:30.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:24:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:24:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:24:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:24:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-20T21:24:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:24:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:24:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:24:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:24:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-20T21:24:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:2 2026-02-20T21:24:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:24:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:24:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-20T21:24:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:2 2026-02-20T21:24:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:24:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:24:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:24:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:24:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-20T21:24:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:24:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:24:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:24:30.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:24:30.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-20T21:24:30.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:24:30.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:24:30.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:24:30.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_demote_image 2026-02-20T21:24:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-20T21:24:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:24:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:24:30.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:24:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_demote_image 2026-02-20T21:24:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:3 2026-02-20T21:24:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:24:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:24:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-20T21:24:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:3 2026-02-20T21:24:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:24:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:24:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:24:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:24:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-20T21:24:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:24:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:24:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:24:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:24:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-02-20T21:24:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:3 2026-02-20T21:24:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:24:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:24:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-20T21:24:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:3 2026-02-20T21:24:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:24:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:24:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:24:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:24:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-20T21:24:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:24:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:24:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:24:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:24:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-20T21:24:30.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:24:30.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:24:30.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:24:30.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_demote_image 2026-02-20T21:24:30.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-20T21:24:30.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:24:30.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:24:30.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:24:30.339 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 1 2026-02-20T21:24:31.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:24:31.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:24:31.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:24:31.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-20T21:24:31.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:24:31.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:24:31.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:24:31.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:24:31.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:24:31.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:24:31.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:24:31.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:24:31.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:24:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:24:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:24:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:24:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:24:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:24:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:24:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:24:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:24:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:24:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:24:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:24:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:24:31.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:24:31.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:24:31.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:24:31.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:24:31.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:24:31.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:24:31.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:24:31.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:24:31.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:24:31.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:24:31.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:24:31.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:24:31.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-20T21:24:31.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:24:31.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:24:31.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:24:31.400 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:24:31.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:24:31.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:24:31.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test_demote_image 2026-02-20T21:24:31.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:31.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:31.402 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test_demote_image started 2026-02-20T21:24:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:24:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:24:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:24:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:24:32.403 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test_demote_image started 2026-02-20T21:24:32.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:32.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:32.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:32.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:24:32.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:24:32.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:24:32.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_demote_image 2026-02-20T21:24:32.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:24:32.404 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:24:32.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:24:32.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:24:32.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:24:32.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:24:32.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:24:32.405 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:24:32.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:24:32.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:24:32.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:24:32.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_demote_image 2026-02-20T21:24:32.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:24:32.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:24:32.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:24:32.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:24:32.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:24:32.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:24:32.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:24:32.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:24:32.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:24:32.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:24:32.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:24:32.410 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:24:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:24:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:24:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:24:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:24:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:24:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:24:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:24:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:24:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:24:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:24:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:24:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:24:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:24:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:24:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:24:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:24:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:24:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:24:32.419 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:24:32.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:24:32.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:24:32.420 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-20T21:24:32.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:24:32.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:24:32.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:24:32.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:24:32.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:24:32.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:24:32.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:24:32.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:24:32.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test_demote_image 2026-02-20T21:24:32.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:24:32.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:32.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:24:32.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:24:32.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:32.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:24:32.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:24:32.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test_demote_image 2026-02-20T21:24:32.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:32.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:32.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:32.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_demote_image 2026-02-20T21:24:33.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 200 2026-02-20T21:24:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test_demote_image 2026-02-20T21:24:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:24:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:24:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:24:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test_demote_image.status 2026-02-20T21:24:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test_demote_image.status 2026-02-20T21:24:33.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:24:33.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_demote_image.status 2026-02-20T21:24:33.012 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test_demote_image.status 2026-02-20T21:24:33.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test_demote_image.status 2026-02-20T21:24:33.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:24:33.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:33.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_demote_image.status 2026-02-20T21:24:33.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:33.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:33.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:33.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_demote_image.status 2026-02-20T21:24:33.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:24:33.015 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:24:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:24:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=200 2026-02-20T21:24:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:24:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:24:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:24:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:24:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:24:33.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:24:33.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=199 2026-02-20T21:24:33.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 200 = 199 2026-02-20T21:24:33.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:24:33.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.4 2026-02-20T21:24:33.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:33.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:33.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:33.677 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:33.678 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:33.678 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:24:33.678 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:24:33.709 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:24:33.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=199 2026-02-20T21:24:33.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 200 = 199 2026-02-20T21:24:33.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:24:33.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.8 2026-02-20T21:24:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:34.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:24:34.513 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:24:34.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:24:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=200 2026-02-20T21:24:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 200 = 200 2026-02-20T21:24:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:24:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+replaying 2026-02-20T21:24:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:34.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:34.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:34.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:24:34.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:24:34.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:24:34.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:24:34.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:24:35.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+replaying '' '' 2026-02-20T21:24:35.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:24:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:24:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:24:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:24:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:24:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-02-20T21:24:35.576 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_demote_image: 2026-02-20T21:24:35.576 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 725c7e75-bf09-4751-a921-33bb6f9b4e5f 2026-02-20T21:24:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:24:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":84650.67,"bytes_per_snapshot":210944.0,"last_snapshot_bytes":421888,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622672,"remote_snapshot_timestamp":1771622672,"replay_state":"idle"} 2026-02-20T21:24:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:24:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:24:35 2026-02-20T21:24:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:24:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:24:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:24:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:24:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:24:34' 2026-02-20T21:24:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:24:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:24:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:24:35.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":84650.67,"bytes_per_snapshot":210944.0,"last_snapshot_bytes":421888,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622672,"remote_snapshot_timestamp":1771622672,"replay_state":"idle"} 2026-02-20T21:24:35.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:24:35.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:24:35.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:24:35.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:24:35.580 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:24:35.581 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:24:35.581 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:24:35.581 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:24:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:24:35' 2026-02-20T21:24:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+replaying '2026-02-20 21:24:35' 2026-02-20T21:24:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:24:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:24:35' 2026-02-20T21:24:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:24:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:24:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:24:35.614 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK161176122mirror.0trial19920.2.0-677-gf7870454trueOK61246130mirror.1trial19920.2.0-677-gf7870454falseOK61316138mirror.2trial19920.2.0-677-gf7870454falseOK61416147mirror.3trial19920.2.0-677-gf7870454falseOKtest_demote_image725c7e75-bf09-4751-a921-33bb6f9b4e5fup+replayingreplaying, {"bytes_per_second":84650.67,"bytes_per_snapshot":210944.0,"last_snapshot_bytes":421888,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622672,"remote_snapshot_timestamp":1771622672,"replay_state":"idle"}61176122mirror.0trial1992026-02-20 21:24:35cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:24:34' 2026-02-20T21:24:35.614 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:24:35.614 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-20T21:24:35.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:24:35' 2026-02-20T21:24:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-20T21:24:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:24:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:24:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:24:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:24:35.619 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:24:35.619 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:24:35.619 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+stopped 2026-02-20T21:24:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:24:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:24:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:24:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:24:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:24:36.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+stopped '' '' 2026-02-20T21:24:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:24:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:24:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:24:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:24:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:24:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-02-20T21:24:36.652 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_demote_image: 2026-02-20T21:24:36.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 725c7e75-bf09-4751-a921-33bb6f9b4e5f 2026-02-20T21:24:36.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:24:36.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:24:36.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:24:36.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:24:34 2026-02-20T21:24:36.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:24:36.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:24:36.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:24:36.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":84650.67,"bytes_per_snapshot":210944.0,"last_snapshot_bytes":421888,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622672,"remote_snapshot_timestamp":1771622672,"replay_state":"idle"} 2026-02-20T21:24:36.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:24:35 2026-02-20T21:24:36.653 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:24:36.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 199 .mirror.primary.725c7e75-bf09-4751-a921-33bb6f9b4e5f.2cad448b-1b27-47a1-9ac0-614f2e99fa6f (peer_uuids:[]) 2026-02-20T21:24:36.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 200 .mirror.primary.725c7e75-bf09-4751-a921-33bb6f9b4e5f.f4ad6e18-0495-4373-accb-99d0986b665a (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:24:36.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:24:36.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:24:36.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:24:36.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:24:36.656 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:24:36.656 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:24:36.656 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:24:36.656 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:24:36.656 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:24:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:24:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:24:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:24:36.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:24:34' 2026-02-20T21:24:36.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+stopped '2026-02-20 21:24:34' 2026-02-20T21:24:36.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:36.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:36.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:36.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:24:36.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:24:34' 2026-02-20T21:24:36.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:24:36.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:24:36.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:24:36.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK163286331mirror.0trial19920.2.0-677-gf7870454trueOK63356338mirror.1trial19920.2.0-677-gf7870454falseOK63426350mirror.2trial19920.2.0-677-gf7870454falseOK63526358mirror.3trial19920.2.0-677-gf7870454falseOKtest_demote_image725c7e75-bf09-4751-a921-33bb6f9b4e5fup+stoppedlocal image is primary63286331mirror.0trial1992026-02-20 21:24:34cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":84650.67,"bytes_per_snapshot":210944.0,"last_snapshot_bytes":421888,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771622672,"remote_snapshot_timestamp":1771622672,"replay_state":"idle"}2026-02-20 21:24:35' 2026-02-20T21:24:36.688 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:24:36.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-20T21:24:36.689 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:24:34' 2026-02-20T21:24:36.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-20T21:24:36.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:24:36.692 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:24:36.692 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:24:36.692 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:24:36.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:24:36.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:24:36.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test_demote_image 2560 4194304 2026-02-20T21:24:36.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:36.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:36.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:36.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=2560 2026-02-20T21:24:36.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=4194304 2026-02-20T21:24:36.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n 4194304 2026-02-20T21:24:36.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test_demote_image --io-type write --io-size 4194304 --io-threads 1 --io-total 10737418240 --io-pattern rand 2026-02-20T21:24:36.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4194304 io_threads 1 bytes 10737418240 pattern random 2026-02-20T21:24:37.731 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:24:37.732 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: 1 567 570.29 2.2 GiB/s 2026-02-20T21:24:38.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: 2 1152 578.824 2.3 GiB/s 2026-02-20T21:24:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: 3 1752 585.905 2.3 GiB/s 2026-02-20T21:24:40.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: 4 2355 590.337 2.3 GiB/s 2026-02-20T21:24:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 4 ops: 2560 ops/sec: 588.785 bytes/sec: 2.3 GiB/s 2026-02-20T21:24:41.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ demote_image cluster2 mirror test_demote_image 2026-02-20T21:24:41.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:41.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:41.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:41.080 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test_demote_image 2026-02-20T21:24:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image demoted to non-primary 2026-02-20T21:24:43.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_complete_mirror_snapshot_id cluster2 mirror test_demote_image primary_snap_id 2026-02-20T21:24:43.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:43.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:43.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:43.029 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local -n _snap_id=primary_snap_id 2026-02-20T21:24:43.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:24:43.030 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '(//snapshots/snapshot[namespace/complete='\''true'\'']/id)[last()]' 2026-02-20T21:24:43.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ _snap_id=201 2026-02-20T21:24:43.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_non_primary_snap_present cluster1 mirror test_demote_image 201 2026-02-20T21:24:43.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:43.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:43.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:43.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snap_id=201 2026-02-20T21:24:43.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:24:43.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:24:43.063 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.1 2026-02-20T21:24:43.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:24:43.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot/namespace[primary_snap_id='\''201'\'']' 2026-02-20T21:24:43.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:24:43.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirrors cluster1 -KILL 2026-02-20T21:24:43.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:43.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig=-KILL 2026-02-20T21:24:43.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:24:43.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:24:43.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:0 -KILL 2026-02-20T21:24:43.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:24:43.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig=-KILL 2026-02-20T21:24:43.205 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:24:43.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:24:43.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:0 2026-02-20T21:24:43.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:0 2026-02-20T21:24:43.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:24:43.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:24:43.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:24:43.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:24:43.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:24:43.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:24:43.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:24:43.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:24:43.206 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:24:43.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:24:43.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:24:43.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:24:43.207 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-20T21:24:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:24:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:24:43.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=84831 2026-02-20T21:24:43.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 84831 ']' 2026-02-20T21:24:43.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill -KILL 84831 2026-02-20T21:24:43.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:24:43.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:24:43.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:24:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=84831 '$2 == pid {print; exit 1}' 2026-02-20T21:24:43.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:24:43.252 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:24:43.253 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=84831 '$2 == pid {print; exit 1}' 2026-02-20T21:24:43.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:24:43.277 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:24:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:24:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:24:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:24:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:24:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:24:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:24:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:24:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:24:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:24:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:24:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:24:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:24:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:24:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:24:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:24:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:24:43.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-20T21:24:43.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:24:43.292 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:24:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:24:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:24:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:24:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:24:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:24:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:24:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:24:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:24:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:24:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:24:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:24:43.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:24:43.294 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:24:43.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:24:43.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:24:43.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:0 2026-02-20T21:24:43.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:24:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:24:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:24:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:24:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:24:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:24:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:24:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:24:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:24:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:24:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:24:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:24:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:24:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-20T21:24:43.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:24:43.317 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:24:43.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:24:43.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:1 -KILL 2026-02-20T21:24:43.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:24:43.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig=-KILL 2026-02-20T21:24:43.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:24:43.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:24:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:1 2026-02-20T21:24:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:1 2026-02-20T21:24:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:24:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:24:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:24:43.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:24:43.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:24:43.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:24:43.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:24:43.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:24:43.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:24:43.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:24:43.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:24:43.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:24:43.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-20T21:24:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:24:43.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:24:43.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=84866 2026-02-20T21:24:43.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 84866 ']' 2026-02-20T21:24:43.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill -KILL 84866 2026-02-20T21:24:43.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:24:43.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:24:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:24:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=84866 '$2 == pid {print; exit 1}' 2026-02-20T21:24:43.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:24:43.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:24:43.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=84866 '$2 == pid {print; exit 1}' 2026-02-20T21:24:43.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:24:43.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:24:43.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:24:43.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:24:43.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:24:43.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:24:43.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:24:43.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:24:43.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:24:43.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:24:43.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:24:43.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:24:43.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:24:43.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:24:43.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:24:43.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:24:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:24:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:24:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-20T21:24:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:24:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:24:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:24:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:24:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:24:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:24:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:24:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:24:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:24:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:24:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:24:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:24:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:24:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:24:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:24:43.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:24:43.409 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:24:43.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:1 2026-02-20T21:24:43.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:24:43.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:24:43.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:24:43.411 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:24:43.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:24:43.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:24:43.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:24:43.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:24:43.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:24:43.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:24:43.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:24:43.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:24:43.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:24:43.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-20T21:24:43.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:24:43.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:24:43.424 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:24:43.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:2 -KILL 2026-02-20T21:24:43.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:24:43.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig=-KILL 2026-02-20T21:24:43.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:24:43.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:24:43.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:2 2026-02-20T21:24:43.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:2 2026-02-20T21:24:43.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:24:43.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:24:43.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:24:43.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:24:43.425 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:24:43.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:24:43.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:24:43.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:24:43.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:24:43.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:24:43.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:24:43.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:24:43.426 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-20T21:24:43.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:24:43.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:24:43.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=84920 2026-02-20T21:24:43.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 84920 ']' 2026-02-20T21:24:43.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill -KILL 84920 2026-02-20T21:24:43.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:24:43.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:24:43.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:24:43.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=84920 '$2 == pid {print; exit 1}' 2026-02-20T21:24:43.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:24:43.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:24:43.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=84920 '$2 == pid {print; exit 1}' 2026-02-20T21:24:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:24:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:24:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:24:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:24:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:24:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:24:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:24:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:24:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:24:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:24:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:24:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:24:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:24:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:24:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:24:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:24:43.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:24:43.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:24:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-20T21:24:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:24:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:24:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:24:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:24:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:24:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:24:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:24:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:24:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:24:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:24:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:24:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:24:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:24:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:24:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:24:43.514 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:24:43.514 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:24:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:2 2026-02-20T21:24:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:24:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:24:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:24:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:24:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:24:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:24:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:24:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:24:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:24:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:24:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:24:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:24:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:24:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-20T21:24:43.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:24:43.527 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:24:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:24:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:3 -KILL 2026-02-20T21:24:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:24:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig=-KILL 2026-02-20T21:24:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:24:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:24:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:3 2026-02-20T21:24:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:3 2026-02-20T21:24:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:24:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:24:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:24:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:24:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:24:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:24:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:24:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:24:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:24:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:24:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:24:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:24:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-20T21:24:43.541 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:24:43.541 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:24:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=85007 2026-02-20T21:24:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 85007 ']' 2026-02-20T21:24:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill -KILL 85007 2026-02-20T21:24:43.543 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:24:43.543 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:24:43.543 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:24:43.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=85007 '$2 == pid {print; exit 1}' 2026-02-20T21:24:43.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:24:43.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:24:43.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=85007 '$2 == pid {print; exit 1}' 2026-02-20T21:24:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:24:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:24:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:24:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:24:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:24:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:24:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:24:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:24:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:24:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:24:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:24:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:24:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:24:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:24:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:24:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:24:43.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:24:43.607 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:24:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-20T21:24:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:24:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:24:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:24:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:24:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:24:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:24:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:24:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:24:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:24:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:24:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:24:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:24:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:24:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:24:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:24:43.620 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:24:43.621 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:24:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:3 2026-02-20T21:24:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:24:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:24:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:24:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:24:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:24:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:24:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:24:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:24:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:24:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:24:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:24:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:24:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:24:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-20T21:24:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:24:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:24:43.635 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_snaps_json cluster1 mirror test_demote_image 2026-02-20T21:24:43.635 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:24:43.635 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:24:43.635 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test_demote_image 2026-02-20T21:24:43.635 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 snap ls mirror/test_demote_image --all --format json 2026-02-20T21:24:43.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ SNAPS='[{"id":212,"name":".mirror.non_primary.725c7e75-bf09-4751-a921-33bb6f9b4e5f.a0d37967-f280-4fc6-96c5-94d0af8af833","size":10737418240,"protected":"false","timestamp":"Fri Feb 20 21:24:33 2026","namespace":{"type":"mirror","state":"non-primary","mirror_peer_uuids":[],"complete":true,"primary_mirror_uuid":"2b58eacb-15a2-47fd-b97e-915fa36056e8","primary_snap_id":200,"last_copied_object_number":2560}},{"id":213,"name":".mirror.non_primary.725c7e75-bf09-4751-a921-33bb6f9b4e5f.d9214ff9-1216-4ce3-b5dd-e04bf2c0385c","size":10737418240,"protected":"false","timestamp":"Fri Feb 20 21:24:42 2026","namespace":{"type":"mirror","state":"demoted","mirror_peer_uuids":["051657c0-214b-4bab-85b7-d4f8366fff9d"],"complete":false,"primary_mirror_uuid":"2b58eacb-15a2-47fd-b97e-915fa36056e8","primary_snap_id":201,"last_copied_object_number":152}}]' 2026-02-20T21:24:43.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ jq -e '.[-1].namespace["type"] == "mirror" and .[-1].namespace["state"] == "demoted" and .[-1].namespace["complete"] == false' 2026-02-20T21:24:43.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:mirrordemoted051657c0-214b-4bab-85b7-d4f8366fff9dfalse2b58eacb-15a2-47fd-b97e-915fa36056e8201135true 2026-02-20T21:24:43.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirrors cluster1 2026-02-20T21:24:43.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:43.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:24:43.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:24:43.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:0 2026-02-20T21:24:43.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:24:43.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:24:43.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:24:43.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:24:43.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:24:43.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:24:43.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:24:43.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:24:43.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:24:43.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:24:43.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:24:43.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:24:43.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:24:43.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:24:43.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:24:43.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:24:43.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:1 2026-02-20T21:24:43.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:24:43.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:24:43.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:24:43.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:24:43.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:24:43.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:24:43.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:24:43.690 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:24:43.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:24:43.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:24:43.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:24:43.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:24:43.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:24:43.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:24:43.691 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:24:43.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:24:43.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:2 2026-02-20T21:24:43.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:24:43.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:24:43.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:24:43.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:2 2026-02-20T21:24:43.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:24:43.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:24:43.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:24:43.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=2 2026-02-20T21:24:43.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:24:43.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:24:43.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:24:43.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=2 2026-02-20T21:24:43.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:24:43.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:24:43.712 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:24:43.730 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:24:43.730 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:3 2026-02-20T21:24:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:24:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:24:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:24:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:3 2026-02-20T21:24:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:24:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:24:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:24:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=3 2026-02-20T21:24:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:24:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:24:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:24:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=3 2026-02-20T21:24:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:24:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:24:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:24:43.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test_demote_image 2026-02-20T21:24:43.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:24:43.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:43.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:24:43.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:24:43.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:43.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test_demote_image.status 2026-02-20T21:24:43.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test_demote_image.status 2026-02-20T21:24:43.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:24:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_demote_image.status 2026-02-20T21:24:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test_demote_image.status 2026-02-20T21:24:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test_demote_image.status 2026-02-20T21:24:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:24:43.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:43.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_demote_image.status 2026-02-20T21:24:43.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:24:43.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:43.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:43.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_demote_image.status 2026-02-20T21:24:43.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:24:43.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:24:43.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:24:43.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=201 2026-02-20T21:24:43.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:24:43.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:24:43.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:24:43.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:43.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:43.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:43.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:43.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:43.985 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:24:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:24:44.017 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:24:44.018 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:24:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:-:1.1: Document is empty 2026-02-20T21:24:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 2026-02-20T21:24:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:^ 2026-02-20T21:24:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id= 2026-02-20T21:24:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 201 = '' 2026-02-20T21:24:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:24:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.4 2026-02-20T21:24:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:44.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:44.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:24:44.422 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:24:44.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:24:44.452 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:24:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:-:1.1: Document is empty 2026-02-20T21:24:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 2026-02-20T21:24:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:^ 2026-02-20T21:24:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id= 2026-02-20T21:24:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 201 = '' 2026-02-20T21:24:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:24:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.8 2026-02-20T21:24:45.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:45.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:45.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:45.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:24:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:24:45.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:24:45.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:24:45.286 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:-:1.1: Document is empty 2026-02-20T21:24:45.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 2026-02-20T21:24:45.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:^ 2026-02-20T21:24:45.287 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id= 2026-02-20T21:24:45.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 201 = '' 2026-02-20T21:24:45.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:24:45.288 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1.6 2026-02-20T21:24:46.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:46.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:46.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:46.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:46.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:46.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:24:46.889 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:24:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:24:46.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:24:46.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:-:1.1: Document is empty 2026-02-20T21:24:46.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 2026-02-20T21:24:46.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:^ 2026-02-20T21:24:46.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id= 2026-02-20T21:24:46.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 201 = '' 2026-02-20T21:24:46.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:24:46.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:24:48.922 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:24:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:24:48.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:24:48.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:24:48.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:-:1.1: Document is empty 2026-02-20T21:24:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 2026-02-20T21:24:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:^ 2026-02-20T21:24:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id= 2026-02-20T21:24:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 201 = '' 2026-02-20T21:24:48.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:24:48.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:24:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:24:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:24:50.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:24:50.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:24:50.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:-:1.1: Document is empty 2026-02-20T21:24:50.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 2026-02-20T21:24:50.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:^ 2026-02-20T21:24:50.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id= 2026-02-20T21:24:50.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 201 = '' 2026-02-20T21:24:50.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:24:50.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:24:54.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:54.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:54.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:54.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:54.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:54.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:24:54.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:24:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:24:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:24:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:-:1.1: Document is empty 2026-02-20T21:24:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 2026-02-20T21:24:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:^ 2026-02-20T21:24:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id= 2026-02-20T21:24:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 201 = '' 2026-02-20T21:24:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:24:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 4 2026-02-20T21:24:59.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:59.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:24:59.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:24:59.027 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:24:59.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:24:59.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:24:59.028 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:24:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:24:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:24:59.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:-:1.1: Document is empty 2026-02-20T21:24:59.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 2026-02-20T21:24:59.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:^ 2026-02-20T21:24:59.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id= 2026-02-20T21:24:59.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 201 = '' 2026-02-20T21:24:59.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:24:59.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 8 2026-02-20T21:25:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:25:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:25:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:25:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:25:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:25:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:25:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:25:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:25:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:25:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:-:1.1: Document is empty 2026-02-20T21:25:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 2026-02-20T21:25:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:^ 2026-02-20T21:25:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id= 2026-02-20T21:25:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 201 = '' 2026-02-20T21:25:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:25:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 8 2026-02-20T21:25:15.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:25:15.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:25:15.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:25:15.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:25:15.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:25:15.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:25:15.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:25:15.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:25:15.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:25:15.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:-:1.1: Document is empty 2026-02-20T21:25:15.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 2026-02-20T21:25:15.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:^ 2026-02-20T21:25:15.130 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id= 2026-02-20T21:25:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 201 = '' 2026-02-20T21:25:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:25:15.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 16 2026-02-20T21:25:31.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:25:31.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:25:31.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:25:31.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:25:31.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:25:31.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:25:31.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:25:31.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:25:31.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:25:31.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:-:1.1: Document is empty 2026-02-20T21:25:31.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 2026-02-20T21:25:31.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:^ 2026-02-20T21:25:31.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id= 2026-02-20T21:25:31.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 201 = '' 2026-02-20T21:25:31.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:25:31.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 16 2026-02-20T21:25:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:25:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:25:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:25:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:25:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.status 2026-02-20T21:25:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-20T21:25:47.166 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:25:47.197 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:25:47.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=201 2026-02-20T21:25:47.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 201 = 201 2026-02-20T21:25:47.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:25:47.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+unknown 2026-02-20T21:25:47.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:25:47.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:25:47.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:25:47.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:25:47.199 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:25:47.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:25:47.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:25:47.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:25:48.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+unknown '' '' 2026-02-20T21:25:48.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:25:48.200 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:25:48.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:25:48.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:25:48.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:25:48.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:25:48.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:25:48.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:25:48.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-02-20T21:25:48.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_demote_image: 2026-02-20T21:25:48.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 725c7e75-bf09-4751-a921-33bb6f9b4e5f 2026-02-20T21:25:48.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:25:48.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:25:48.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:25:48.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:25:44 2026-02-20T21:25:48.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:25:48.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:25:48.231 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:25:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:25:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:25:43' 2026-02-20T21:25:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:25:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:25:48.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:25:48.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:25:48.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:25:48.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:25:48.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:25:48.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:25:48.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:25:48.235 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:25:48.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:25:48.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:25:48.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:25:44' 2026-02-20T21:25:48.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+unknown '2026-02-20 21:25:44' 2026-02-20T21:25:48.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:25:48.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:25:48.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:25:48.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:25:48.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:25:44' 2026-02-20T21:25:48.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:25:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:25:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:25:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING163286331mirror.0trial19920.2.0-677-gf7870454trueOK63356338mirror.1trial19920.2.0-677-gf7870454falseOK63426350mirror.2trial19920.2.0-677-gf7870454falseOK63526358mirror.3trial19920.2.0-677-gf7870454falseOKtest_demote_image725c7e75-bf09-4751-a921-33bb6f9b4e5fup+unknownremote image is not primary63286331mirror.0trial1992026-02-20 21:25:44cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+unknownremote image is not primary2026-02-20 21:25:43' 2026-02-20T21:25:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:25:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-20T21:25:48.267 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:25:44' 2026-02-20T21:25:48.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-20T21:25:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:25:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:25:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:25:48.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:25:48.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:25:48.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:25:48.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+unknown 2026-02-20T21:25:48.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:25:48.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:25:48.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:25:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:25:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:25:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:25:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:25:48.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:25:49.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+unknown '' '' 2026-02-20T21:25:49.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:25:49.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:25:49.272 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:25:49.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:25:49.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:25:49.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:25:49.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:25:49.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:25:49.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-02-20T21:25:49.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_demote_image: 2026-02-20T21:25:49.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 725c7e75-bf09-4751-a921-33bb6f9b4e5f 2026-02-20T21:25:49.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:25:49.303 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:25:49.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:25:49.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:25:48 2026-02-20T21:25:49.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:25:49.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:25:49.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+unknown 2026-02-20T21:25:49.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: remote image is not primary 2026-02-20T21:25:49.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:25:49' 2026-02-20T21:25:49.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+unknown' 2026-02-20T21:25:49.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+unknown 2026-02-20T21:25:49.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:25:49.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: remote image is not primary 2026-02-20T21:25:49.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:25:49.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:25:49.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:25:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:25:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:25:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:25:49.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:25:49.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:25:49.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:25:48' 2026-02-20T21:25:49.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+unknown '2026-02-20 21:25:48' 2026-02-20T21:25:49.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:25:49.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:25:49.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:25:49.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+unknown 2026-02-20T21:25:49.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:25:48' 2026-02-20T21:25:49.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:25:49.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:25:49.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:25:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='WARNINGOKWARNING162946297mirror.0trial19920.2.0-677-gf7870454falseOK62996303mirror.1trial19920.2.0-677-gf7870454trueOK63056312mirror.2trial19920.2.0-677-gf7870454falseOK63146317mirror.3trial19920.2.0-677-gf7870454falseOKtest_demote_image725c7e75-bf09-4751-a921-33bb6f9b4e5fup+unknownremote image is not primary62946297mirror.0trial1992026-02-20 21:25:48cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+unknownremote image is not primary2026-02-20 21:25:49' 2026-02-20T21:25:49.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:25:49.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-20T21:25:49.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:25:48' 2026-02-20T21:25:49.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-20T21:25:49.345 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+unknown 2026-02-20T21:25:49.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+unknown 2026-02-20T21:25:49.346 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+unknown 2026-02-20T21:25:49.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+unknown 2026-02-20T21:25:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:25:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:25:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ compare_images cluster1 cluster2 mirror mirror test_demote_image 2026-02-20T21:25:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local ret=0 2026-02-20T21:25:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:25:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:25:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:25:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:25:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:25:49.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test_demote_image.export 2026-02-20T21:25:49.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test_demote_image.export 2026-02-20T21:25:49.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:25:49.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local rmt_export=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_demote_image.export 2026-02-20T21:25:49.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test_demote_image.export 2026-02-20T21:25:49.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test_demote_image.export 2026-02-20T21:25:49.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:25:49.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local loc_export=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.export 2026-02-20T21:25:49.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_demote_image.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.export 2026-02-20T21:25:49.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 export mirror/test_demote_image /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_demote_image.export 2026-02-20T21:25:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:25:53.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 export mirror/test_demote_image /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.export 2026-02-20T21:25:58.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:25:58.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cmp /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_demote_image.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.export 2026-02-20T21:26:03.579 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_demote_image.export /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_demote_image.export 2026-02-20T21:26:05.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:26:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test_demote_image 2026-02-20T21:26:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:26:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:26:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force= 2026-02-20T21:26:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_demote_image 2026-02-20T21:26:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:26:06.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster2 mirror test_demote_image 2026-02-20T21:26:06.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:06.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:06.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:26:06.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test_demote_image started 2026-02-20T21:26:06.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:06.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:06.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:26:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:26:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:26:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:26:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:26:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test_demote_image started 2026-02-20T21:26:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:26:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:26:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:26:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:26:07.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-02-20T21:26:07.054 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:26:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:26:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:26:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:26:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:26:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:26:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:26:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:26:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:26:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:26:07.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:26:07.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-20T21:26:07.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:26:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:26:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:26:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:26:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:26:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:26:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:26:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:26:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:26:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:26:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:26:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:26:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:26:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:26:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:26:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:26:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:26:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:26:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:26:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:26:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:26:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:26:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:26:07.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:26:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:26:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:26:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:26:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:26:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:26:07.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:26:07.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:26:07.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:26:07.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:26:07.070 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-20T21:26:07.115 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:26:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:26:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:26:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:26:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = stopped 2026-02-20T21:26:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:26:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 2 2026-02-20T21:26:09.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test_demote_image started 2026-02-20T21:26:09.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:09.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:09.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:26:09.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:26:09.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:26:09.118 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:26:09.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-02-20T21:26:09.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:26:09.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:26:09.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:26:09.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:26:09.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:26:09.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:26:09.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:26:09.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:26:09.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:26:09.120 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:26:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:26:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-20T21:26:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:26:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:26:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:26:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:26:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:26:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:26:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:26:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:26:09.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:26:09.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:26:09.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:26:09.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:26:09.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:26:09.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:26:09.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:26:09.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:26:09.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:26:09.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:26:09.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:26:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:26:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:26:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:26:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:26:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:26:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:26:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:26:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:26:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:26:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:26:09.125 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:26:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:26:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:26:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:26:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-20T21:26:09.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:26:09.179 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:26:09.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:26:09.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:26:09.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:26:09.181 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:26:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:26:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:26:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+stopped 2026-02-20T21:26:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:26:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:26:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:26:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:26:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:26:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:26:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:26:10.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+stopped '' '' 2026-02-20T21:26:10.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:26:10.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:10.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:26:10.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:26:10.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:26:10.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:26:10.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:26:10.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:26:10.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-02-20T21:26:10.237 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_demote_image: 2026-02-20T21:26:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 725c7e75-bf09-4751-a921-33bb6f9b4e5f 2026-02-20T21:26:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:26:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:26:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:26:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:26:08 2026-02-20T21:26:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:26:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:26:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:26:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622766,"replay_state":"idle"} 2026-02-20T21:26:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:26:09 2026-02-20T21:26:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:26:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 214 .mirror.primary.725c7e75-bf09-4751-a921-33bb6f9b4e5f.4013c471-ac19-4ef9-8203-f4d6d88b401d (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:26:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:26:10.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:26:10.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:26:10.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:26:10.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:26:10.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:26:10.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:26:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:26:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:26:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:26:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:26:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:26:10.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:26:08' 2026-02-20T21:26:10.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+stopped '2026-02-20 21:26:08' 2026-02-20T21:26:10.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:26:10.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:10.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:26:10.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:26:10.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:26:08' 2026-02-20T21:26:10.246 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:26:10.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:26:10.247 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:26:10.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK162946297mirror.0trial19920.2.0-677-gf7870454falseOK62996303mirror.1trial19920.2.0-677-gf7870454trueOK63056312mirror.2trial19920.2.0-677-gf7870454falseOK63146317mirror.3trial19920.2.0-677-gf7870454falseOKtest_demote_image725c7e75-bf09-4751-a921-33bb6f9b4e5fup+stoppedlocal image is primary62946297mirror.0trial1992026-02-20 21:26:08cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622766,"replay_state":"idle"}2026-02-20 21:26:09' 2026-02-20T21:26:10.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:26:10.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-20T21:26:10.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:26:08' 2026-02-20T21:26:10.302 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-20T21:26:10.304 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:26:10.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:26:10.305 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:26:10.306 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:26:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:26:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:26:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+replaying 2026-02-20T21:26:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:26:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:26:10.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:26:10.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:26:10.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:26:10.308 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:26:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+replaying '' '' 2026-02-20T21:26:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:26:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:26:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:26:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:26:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:26:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:26:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-02-20T21:26:11.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_demote_image: 2026-02-20T21:26:11.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 725c7e75-bf09-4751-a921-33bb6f9b4e5f 2026-02-20T21:26:11.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:26:11.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622766,"replay_state":"idle"} 2026-02-20T21:26:11.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:26:11.377 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:26:09 2026-02-20T21:26:11.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:26:11.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:26:11.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:26:11.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:26:11.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:26:08' 2026-02-20T21:26:11.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:26:11.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:26:11.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:26:11.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622766,"replay_state":"idle"} 2026-02-20T21:26:11.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:26:11.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:26:11.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:26:11.380 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:26:11.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:26:11.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:26:11.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:26:11.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:26:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:26:09' 2026-02-20T21:26:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+replaying '2026-02-20 21:26:09' 2026-02-20T21:26:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:26:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:26:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:26:09' 2026-02-20T21:26:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:26:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:26:11.388 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:26:11.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK163286331mirror.0trial19920.2.0-677-gf7870454trueOK63356338mirror.1trial19920.2.0-677-gf7870454falseOK63426350mirror.2trial19920.2.0-677-gf7870454falseOK63526358mirror.3trial19920.2.0-677-gf7870454falseOKtest_demote_image725c7e75-bf09-4751-a921-33bb6f9b4e5fup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622766,"replay_state":"idle"}63286331mirror.0trial1992026-02-20 21:26:09cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:26:08' 2026-02-20T21:26:11.460 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:26:11.461 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-20T21:26:11.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:26:09' 2026-02-20T21:26:11.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-20T21:26:11.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:26:11.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:26:11.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:26:11.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:26:11.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:26:11.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:26:11.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster1 mirror test_demote_image 2026-02-20T21:26:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:26:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:26:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:26:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:26:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster1 mirror test_demote_image 2026-02-20T21:26:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:26:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_demote_image 2026-02-20T21:26:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_demote_image 2026-02-20T21:26:11.518 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 rm mirror/test_demote_image 2026-02-20T21:26:12.063 INFO:tasks.ceph.cluster2.osd.0.trial199.stderr:2026-02-20T21:26:12.060+0000 7f169e750640 -1 reset not still connected to 0x557cb6ab7c70 2026-02-20T21:26:12.063 INFO:tasks.ceph.cluster2.osd.0.trial199.stderr:2026-02-20T21:26:12.060+0000 7f169e750640 -1 reset not still connected to 0x557cb6b068f0 2026-02-20T21:26:13.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T21:26:13.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:26:13.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ testlog 'TEST: force promote with a user snapshot' 2026-02-20T21:26:13.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ tee -a /tmp/tmp.2FjbyoErqN/rbd-mirror.test.log 2026-02-20T21:26:13.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ date '+%F %T' 2026-02-20T21:26:13.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo 2026-02-20 21:26:13 TEST: force promote with a user snapshot 2026-02-20T21:26:13.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:2026-02-20 21:26:13 TEST: force promote with a user snapshot 2026-02-20T21:26:13.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ force_promote_image=test_force_promote_user 2026-02-20T21:26:13.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image_and_enable_mirror cluster2 mirror test_force_promote_user snapshot 10G 2026-02-20T21:26:13.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:13.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:26:13.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:13.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:26:13.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:13.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:26:13.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:26:13.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n snapshot ']' 2026-02-20T21:26:13.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:26:13.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_image cluster2 mirror test_force_promote_user 10G 2026-02-20T21:26:13.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:13.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:26:13.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:13.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:26:13.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:13.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:26:13.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=128 2026-02-20T21:26:13.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 10G ']' 2026-02-20T21:26:13.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=10G 2026-02-20T21:26:13.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ shift 2026-02-20T21:26:13.465 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 create --size 10G --image-feature layering,exclusive-lock,object-map,fast-diff mirror/test_force_promote_user 2026-02-20T21:26:13.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:26:13.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:13.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' image = image ']' 2026-02-20T21:26:13.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:26:13.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ enable_mirror cluster2 mirror test_force_promote_user snapshot 2026-02-20T21:26:13.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:13.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:13.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:13.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local mode=snapshot 2026-02-20T21:26:13.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test_force_promote_user snapshot 2026-02-20T21:26:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Mirroring enabled 2026-02-20T21:26:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 info mirror/test_force_promote_user 2026-02-20T21:26:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test_force_promote_user': 2026-02-20T21:26:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 10 GiB in 2560 objects 2026-02-20T21:26:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:26:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 1 2026-02-20T21:26:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 19311fbab833 2026-02-20T21:26:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.19311fbab833 2026-02-20T21:26:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:26:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:26:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:26:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:26:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:26:13 2026 2026-02-20T21:26:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:26:13 2026 2026-02-20T21:26:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:26:13 2026 2026-02-20T21:26:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:26:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:26:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 632eca39-aab5-4396-b3b7-5b0733f0e4d3 2026-02-20T21:26:14.067 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: true 2026-02-20T21:26:14.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test_force_promote_user 100 2026-02-20T21:26:14.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:14.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:14.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:14.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:26:14.068 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:26:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:26:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:26:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote_user --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:26:14.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:26:14.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:26:14.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 2083.36 bytes/sec: 8.1 MiB/s 2026-02-20T21:26:14.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_force_promote_user 2026-02-20T21:26:14.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:14.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:14.151 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:14.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test_force_promote_user stopped 2026-02-20T21:26:14.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:14.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:14.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:14.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:26:14.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:26:14.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:26:14.152 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:26:15.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test_force_promote_user stopped 2026-02-20T21:26:15.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:15.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:15.153 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:15.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:26:15.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:26:15.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:26:15.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:26:15.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:26:15.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:26:15.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:26:15.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:26:15.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:26:15.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:26:15.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:26:15.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:26:15.155 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:26:15.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:26:15.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:26:15.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:26:15.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:26:15.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:26:15.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:26:15.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:26:15.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:26:15.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:26:15.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:26:15.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:26:15.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:26:15.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:26:15.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:26:15.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:26:15.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:26:15.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:26:15.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:26:15.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:26:15.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:26:15.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:26:15.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:26:15.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:26:15.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:26:15.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:26:15.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:26:15.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:26:15.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:26:15.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:26:15.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:26:15.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:26:15.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:26:15.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:26:15.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:26:15.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:26:15.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:26:15.170 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:26:15.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:26:15.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:26:15.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:26:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:26:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:26:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:26:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:26:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:26:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:26:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:26:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:26:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:26:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:26:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:26:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:26:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:26:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:26:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:26:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:26:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:26:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:26:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:26:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:26:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:26:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:26:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:26:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:26:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:26:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:26:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:26:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:26:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:26:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:26:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:26:15.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:26:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:26:15.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:26:15.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:26:15.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:26:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:26:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:26:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:26:15.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:26:15.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:26:15.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:2 2026-02-20T21:26:15.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:26:15.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:26:15.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-20T21:26:15.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:2 2026-02-20T21:26:15.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:26:15.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:26:15.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:26:15.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:26:15.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-20T21:26:15.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:26:15.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:26:15.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:26:15.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:26:15.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-20T21:26:15.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:2 2026-02-20T21:26:15.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:26:15.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:26:15.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-20T21:26:15.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:2 2026-02-20T21:26:15.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:26:15.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:26:15.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:26:15.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:26:15.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-20T21:26:15.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:26:15.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:26:15.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:26:15.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:26:15.276 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-20T21:26:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:26:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:26:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:26:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:26:15.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:26:15.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:26:15.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:26:15.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:26:15.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:26:15.330 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:3 2026-02-20T21:26:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:26:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:26:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-20T21:26:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:3 2026-02-20T21:26:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:26:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:26:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:26:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:26:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-20T21:26:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:26:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:26:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:26:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:26:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-02-20T21:26:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:3 2026-02-20T21:26:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:26:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:26:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-20T21:26:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:3 2026-02-20T21:26:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:26:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:26:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:26:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:26:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-20T21:26:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:26:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:26:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:26:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:26:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-20T21:26:15.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:26:15.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:26:15.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:26:15.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:26:15.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:26:15.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:26:15.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:26:15.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:26:15.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 1 2026-02-20T21:26:16.288 INFO:tasks.ceph.cluster2.osd.0.trial199.stderr:2026-02-20T21:26:16.285+0000 7f169e750640 -1 reset not still connected to 0x557cb83b68f0 2026-02-20T21:26:16.288 INFO:tasks.ceph.cluster2.osd.0.trial199.stderr:2026-02-20T21:26:16.285+0000 7f169e750640 -1 reset not still connected to 0x557cba1028f0 2026-02-20T21:26:16.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:26:16.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:26:16.389 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:26:16.389 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:26:16.389 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:26:16.389 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:26:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:26:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:26:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:26:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:26:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:26:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:26:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:26:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:26:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:26:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:26:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:26:16.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:26:16.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:26:16.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:26:16.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:26:16.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:26:16.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:26:16.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:26:16.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:26:16.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:26:16.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:26:16.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:26:16.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:26:16.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:26:16.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:26:16.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:26:16.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:26:16.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:26:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:26:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:26:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:26:16.407 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:26:16.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:26:16.462 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:26:16.463 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:26:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:26:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:26:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:26:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:26:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:26:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:26:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:26:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:26:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:26:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:26:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:26:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:26:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:26:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:26:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:26:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:26:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:26:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:26:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:26:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:26:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:26:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:26:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:26:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:26:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:26:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:26:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:26:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:26:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:26:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:26:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:26:16.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:26:16.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:26:16.482 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:26:16.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:26:16.483 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:26:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:26:16.550 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:26:16.550 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:26:16.550 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:26:16.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:26:16.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:26:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_started cluster1 mirror test_force_promote_user 2026-02-20T21:26:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:26:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test_force_promote_user started 2026-02-20T21:26:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:26:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=started 2026-02-20T21:26:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:26:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:26:16.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:26:16.778 INFO:tasks.ceph.cluster2.osd.0.trial199.stderr:2026-02-20T21:26:16.776+0000 7f169e750640 -1 reset not still connected to 0x557cb83b68f0 2026-02-20T21:26:16.778 INFO:tasks.ceph.cluster2.osd.0.trial199.stderr:2026-02-20T21:26:16.776+0000 7f169e750640 -1 reset not still connected to 0x557cba1028f0 2026-02-20T21:26:17.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test_force_promote_user started 2026-02-20T21:26:17.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:26:17.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:17.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:17.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=started 2026-02-20T21:26:17.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:26:17.554 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:26:17.555 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:26:17.555 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:26:17.555 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:26:17.555 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:26:17.555 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:26:17.555 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:26:17.555 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:26:17.556 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:26:17.556 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:26:17.556 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:26:17.556 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:26:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:26:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:26:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:26:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:26:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:26:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:26:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:26:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:26:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:26:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:26:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:26:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:26:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:26:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:26:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:26:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:26:17.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:26:17.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:26:17.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:26:17.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:26:17.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:26:17.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:26:17.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:26:17.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:26:17.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:26:17.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:26:17.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:26:17.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:26:17.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:26:17.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:26:17.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:26:17.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:26:17.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:26:17.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:26:17.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:26:17.571 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:26:17.611 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:26:17.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:26:17.612 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:26:17.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:26:17.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:26:17.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:26:17.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:26:17.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:26:17.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:26:17.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:26:17.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:26:17.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:26:17.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:26:17.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:26:17.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:26:17.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:26:17.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:26:17.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:26:17.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:26:17.616 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:26:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:26:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:26:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:26:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:26:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:26:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:26:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:26:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:26:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:26:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:26:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:26:17.618 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:26:17.618 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:26:17.618 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:26:17.618 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:26:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:26:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:26:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:26:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:26:17.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:26:17.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:26:17.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:26:17.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:26:17.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: "state": "Replaying" 2026-02-20T21:26:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ current_state=started 2026-02-20T21:26:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test started = started 2026-02-20T21:26:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:26:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test_force_promote_user 2026-02-20T21:26:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:26:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:26:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:26:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:26:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:26:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test_force_promote_user 2026-02-20T21:26:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_force_promote_user 2026-02-20T21:26:18.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 204 2026-02-20T21:26:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test_force_promote_user 2026-02-20T21:26:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_cluster=cluster1 2026-02-20T21:26:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_pool=mirror 2026-02-20T21:26:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local remote_pool=mirror 2026-02-20T21:26:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster2-mirror-test_force_promote_user.status 2026-02-20T21:26:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster2-mirror-test_force_promote_user.status 2026-02-20T21:26:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:26:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_force_promote_user.status 2026-02-20T21:26:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ mkfname cluster1-mirror-test_force_promote_user.status 2026-02-20T21:26:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo cluster1-mirror-test_force_promote_user.status 2026-02-20T21:26:18.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:26:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local local_status_log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_force_promote_user.status 2026-02-20T21:26:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_force_promote_user /tmp/tmp.2FjbyoErqN/cluster2-mirror-test_force_promote_user.status 2026-02-20T21:26:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster2-mirror-test_force_promote_user.status 2026-02-20T21:26:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_force_promote_user --format xml 2026-02-20T21:26:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:26:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:26:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snapshot_id=204 2026-02-20T21:26:18.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ true 2026-02-20T21:26:18.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:26:18.083 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.2 2026-02-20T21:26:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_force_promote_user /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_force_promote_user.status 2026-02-20T21:26:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:26:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_force_promote_user.status 2026-02-20T21:26:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_force_promote_user --format xml 2026-02-20T21:26:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:26:18.316 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:26:18.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=203 2026-02-20T21:26:18.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 204 = 203 2026-02-20T21:26:18.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:26:18.318 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.4 2026-02-20T21:26:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_force_promote_user /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_force_promote_user.status 2026-02-20T21:26:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:26:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_force_promote_user.status 2026-02-20T21:26:18.720 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_force_promote_user --format xml 2026-02-20T21:26:18.720 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:26:18.750 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:26:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=203 2026-02-20T21:26:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 204 = 203 2026-02-20T21:26:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:26:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0.8 2026-02-20T21:26:19.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_force_promote_user /tmp/tmp.2FjbyoErqN/cluster1-mirror-test_force_promote_user.status 2026-02-20T21:26:19.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:26:19.554 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:19.554 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:19.554 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local log=/tmp/tmp.2FjbyoErqN/cluster1-mirror-test_force_promote_user.status 2026-02-20T21:26:19.554 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_force_promote_user --format xml 2026-02-20T21:26:19.554 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:26:19.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:26:19.590 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local primary_snapshot_id=204 2026-02-20T21:26:19.590 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test 204 = 204 2026-02-20T21:26:19.590 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:26:19.590 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_force_promote_user 2026-02-20T21:26:19.590 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:26:19.590 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:19.590 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:19.591 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:26:19.591 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_force_promote_user up+replaying 2026-02-20T21:26:19.591 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:26:19.591 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:19.591 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:19.591 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:26:19.591 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:26:19.591 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:26:19.591 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:26:19.591 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:26:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_force_promote_user up+replaying '' '' 2026-02-20T21:26:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:26:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:26:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:26:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:26:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:26:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:26:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_force_promote_user 2026-02-20T21:26:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_force_promote_user: 2026-02-20T21:26:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 632eca39-aab5-4396-b3b7-5b0733f0e4d3 2026-02-20T21:26:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:26:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622774,"replay_state":"idle"} 2026-02-20T21:26:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:26:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:26:15 2026-02-20T21:26:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:26:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:26:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:26:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:26:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:26:19' 2026-02-20T21:26:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:26:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+replaying 2026-02-20T21:26:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:26:20.624 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622774,"replay_state":"idle"} 2026-02-20T21:26:20.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:26:20.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:26:20.625 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:26:20.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:26:20.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:26:20.627 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:26:20.627 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:26:20.627 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:26:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:26:15' 2026-02-20T21:26:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_force_promote_user up+replaying '2026-02-20 21:26:15' 2026-02-20T21:26:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:26:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+replaying 2026-02-20T21:26:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:26:15' 2026-02-20T21:26:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:26:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:26:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:26:20.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK162946297mirror.0trial19920.2.0-677-gf7870454falseOK62996303mirror.1trial19920.2.0-677-gf7870454trueOK63056312mirror.2trial19920.2.0-677-gf7870454falseOK63146317mirror.3trial19920.2.0-677-gf7870454falseOKtest_force_promote_user632eca39-aab5-4396-b3b7-5b0733f0e4d3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622774,"replay_state":"idle"}62996303mirror.1trial1992026-02-20 21:26:15cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:26:19' 2026-02-20T21:26:20.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:26:20.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/last_update' 2026-02-20T21:26:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:26:15' 2026-02-20T21:26:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/state' 2026-02-20T21:26:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+replaying 2026-02-20T21:26:20.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+replaying 2026-02-20T21:26:20.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+replaying 2026-02-20T21:26:20.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+replaying 2026-02-20T21:26:20.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:26:20.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:26:20.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_force_promote_user up+stopped 2026-02-20T21:26:20.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:20.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:20.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:20.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:26:20.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:26:20.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:26:20.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:26:20.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:26:21.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test_force_promote_user up+stopped '' '' 2026-02-20T21:26:21.667 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:21.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:21.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:21.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:26:21.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:26:21.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:26:21.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:26:21.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:26:21.668 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_force_promote_user 2026-02-20T21:26:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_force_promote_user: 2026-02-20T21:26:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 632eca39-aab5-4396-b3b7-5b0733f0e4d3 2026-02-20T21:26:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:26:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:26:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:26:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:26:19 2026-02-20T21:26:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:26:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:26:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+replaying 2026-02-20T21:26:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622774,"replay_state":"idle"} 2026-02-20T21:26:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:26:15 2026-02-20T21:26:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:26:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 203 .mirror.primary.632eca39-aab5-4396-b3b7-5b0733f0e4d3.bf2d6cf1-7cc5-4a1b-a964-8fefc9b42cc1 (peer_uuids:[]) 2026-02-20T21:26:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 204 .mirror.primary.632eca39-aab5-4396-b3b7-5b0733f0e4d3.7a67a5d0-629a-41e5-a8f8-0ae0a0b0987b (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:26:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:26:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:26:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:26:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:26:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:26:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:26:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:26:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:26:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:26:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:26:21.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:26:21.703 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:26:21.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:26:19' 2026-02-20T21:26:21.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_force_promote_user up+stopped '2026-02-20 21:26:19' 2026-02-20T21:26:21.704 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:21.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:21.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:21.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:26:21.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:26:19' 2026-02-20T21:26:21.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:26:21.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:26:21.705 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:26:21.734 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK163286331mirror.0trial19920.2.0-677-gf7870454trueOK63356338mirror.1trial19920.2.0-677-gf7870454falseOK63426350mirror.2trial19920.2.0-677-gf7870454falseOK63526358mirror.3trial19920.2.0-677-gf7870454falseOKtest_force_promote_user632eca39-aab5-4396-b3b7-5b0733f0e4d3up+stoppedlocal image is primary63356338mirror.1trial1992026-02-20 21:26:19cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771622774,"replay_state":"idle"}2026-02-20 21:26:15' 2026-02-20T21:26:21.734 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:26:21.734 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/last_update' 2026-02-20T21:26:21.735 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:26:19' 2026-02-20T21:26:21.736 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/state' 2026-02-20T21:26:21.737 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:26:21.738 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:26:21.738 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:26:21.739 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:26:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:26:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:26:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test_force_promote_user 100 2026-02-20T21:26:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:26:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:26:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:26:21.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:26:21.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote_user --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:26:21.770 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:26:21.807 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:26:21.808 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 2631.62 bytes/sec: 10 MiB/s 2026-02-20T21:26:21.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ create_snapshot cluster2 mirror test_force_promote_user snap1 2026-02-20T21:26:21.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:21.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:21.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:21.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap=snap1 2026-02-20T21:26:21.813 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 snap create mirror/test_force_promote_user@snap1 2026-02-20T21:26:22.039 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-20T21:26:22.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test_force_promote_user 2560 4194304 2026-02-20T21:26:22.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:22.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:22.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:22.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=2560 2026-02-20T21:26:22.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size=4194304 2026-02-20T21:26:22.045 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n 4194304 2026-02-20T21:26:22.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote_user --io-type write --io-size 4194304 --io-threads 1 --io-total 10737418240 --io-pattern rand 2026-02-20T21:26:22.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4194304 io_threads 1 bytes 10737418240 pattern random 2026-02-20T21:26:23.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:26:23.082 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: 1 602 610.333 2.4 GiB/s 2026-02-20T21:26:24.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: 2 1210 609.778 2.4 GiB/s 2026-02-20T21:26:25.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: 3 1822 610.525 2.4 GiB/s 2026-02-20T21:26:26.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: 4 2453 615.664 2.4 GiB/s 2026-02-20T21:26:26.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 4 ops: 2560 ops/sec: 612.596 bytes/sec: 2.4 GiB/s 2026-02-20T21:26:26.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:26:26.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ mirror_image_snapshot cluster2 mirror test_force_promote_user 2026-02-20T21:26:26.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:26:26.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:26.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:26.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test_force_promote_user 2026-02-20T21:26:28.180 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Snapshot ID: 206 2026-02-20T21:26:28.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_snap_present cluster1 mirror test_force_promote_user snap1 2026-02-20T21:26:28.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:26:28.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:28.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:28.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local snap_name=snap1 2026-02-20T21:26:28.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:26:28.189 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:26:28.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:26:29.190 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 info mirror/test_force_promote_user@snap1 2026-02-20T21:26:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:rbd image 'test_force_promote_user': 2026-02-20T21:26:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: size 10 GiB in 2560 objects 2026-02-20T21:26:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: order 22 (4 MiB objects) 2026-02-20T21:26:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: snapshot_count: 3 2026-02-20T21:26:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: id: 189fb3608bbc 2026-02-20T21:26:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: block_name_prefix: rbd_data.189fb3608bbc 2026-02-20T21:26:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: format: 2 2026-02-20T21:26:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:26:29.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: op_features: 2026-02-20T21:26:29.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: flags: 2026-02-20T21:26:29.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: create_timestamp: Fri Feb 20 21:26:15 2026 2026-02-20T21:26:29.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: access_timestamp: Fri Feb 20 21:26:15 2026 2026-02-20T21:26:29.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: modify_timestamp: Fri Feb 20 21:26:15 2026 2026-02-20T21:26:29.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: protected: False 2026-02-20T21:26:29.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring state: enabled 2026-02-20T21:26:29.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring mode: snapshot 2026-02-20T21:26:29.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring global id: 632eca39-aab5-4396-b3b7-5b0733f0e4d3 2026-02-20T21:26:29.234 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: mirroring primary: false 2026-02-20T21:26:29.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:26:29.236 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:26:30.145 INFO:tasks.ceph.cluster2.osd.2.trial199.stderr:2026-02-20T21:26:30.142+0000 7fbfd8a6f640 -1 reset not still connected to 0x563141191c70 2026-02-20T21:26:30.145 INFO:tasks.ceph.cluster2.osd.2.trial199.stderr:2026-02-20T21:26:30.142+0000 7fbfd8a6f640 -1 reset not still connected to 0x563141d92820 2026-02-20T21:26:30.145 INFO:tasks.ceph.cluster2.osd.2.trial199.stderr:2026-02-20T21:26:30.142+0000 7fbfd8a6f640 -1 reset not still connected to 0x563141d935f0 2026-02-20T21:26:30.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirrors cluster1 -KILL 2026-02-20T21:26:30.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:26:30.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig=-KILL 2026-02-20T21:26:30.238 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:26:30.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:26:30.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:0 -KILL 2026-02-20T21:26:30.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:26:30.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig=-KILL 2026-02-20T21:26:30.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:26:30.239 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:26:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:0 2026-02-20T21:26:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:0 2026-02-20T21:26:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:26:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:26:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:26:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:26:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:26:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:26:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:26:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:26:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:26:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:26:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:26:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:26:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-20T21:26:30.257 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:26:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:26:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=88010 2026-02-20T21:26:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 88010 ']' 2026-02-20T21:26:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill -KILL 88010 2026-02-20T21:26:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:26:30.260 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:26:30.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:26:30.262 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=88010 '$2 == pid {print; exit 1}' 2026-02-20T21:26:30.297 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:26:30.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:26:30.298 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=88010 '$2 == pid {print; exit 1}' 2026-02-20T21:26:30.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:26:30.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:26:30.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:26:30.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:26:30.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:26:30.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:26:30.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:26:30.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:26:30.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:26:30.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:26:30.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:26:30.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:26:30.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:26:30.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:26:30.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:26:30.329 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:26:30.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:26:30.340 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:26:30.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-20T21:26:30.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:26:30.342 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:26:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:26:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:26:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:26:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:26:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:26:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:26:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:26:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:26:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:26:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:26:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:26:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:26:30.344 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:26:30.354 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:26:30.355 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:26:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:0 2026-02-20T21:26:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:26:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:26:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:26:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:26:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:26:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:26:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:26:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:26:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:26:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:26:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:26:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:26:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:26:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-20T21:26:30.368 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:26:30.369 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:26:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:26:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:1 -KILL 2026-02-20T21:26:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:26:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig=-KILL 2026-02-20T21:26:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:26:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:26:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:1 2026-02-20T21:26:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:1 2026-02-20T21:26:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:26:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:26:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:26:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:26:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:26:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:26:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:26:30.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:26:30.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:26:30.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:26:30.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:26:30.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:26:30.373 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-20T21:26:30.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:26:30.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:26:30.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=88054 2026-02-20T21:26:30.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 88054 ']' 2026-02-20T21:26:30.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill -KILL 88054 2026-02-20T21:26:30.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:26:30.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:26:30.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:26:30.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=88054 '$2 == pid {print; exit 1}' 2026-02-20T21:26:30.412 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:26:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:26:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=88054 '$2 == pid {print; exit 1}' 2026-02-20T21:26:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:26:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:26:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:26:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:26:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:26:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:26:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:26:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:26:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:26:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:26:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:26:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:26:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:26:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:26:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:26:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:26:30.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:26:30.451 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:26:30.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-20T21:26:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:26:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:26:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:26:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:26:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:26:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:26:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:26:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:26:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:26:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:26:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:26:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:26:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:26:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:26:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:26:30.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:26:30.464 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:26:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:1 2026-02-20T21:26:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:26:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:26:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:26:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:26:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:26:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:26:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:26:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:26:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:26:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:26:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:26:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:26:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:26:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-20T21:26:30.476 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:26:30.477 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:26:30.478 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:26:30.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:2 -KILL 2026-02-20T21:26:30.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:26:30.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig=-KILL 2026-02-20T21:26:30.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:26:30.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:26:30.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:2 2026-02-20T21:26:30.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:2 2026-02-20T21:26:30.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:26:30.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:26:30.479 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:26:30.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:26:30.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:26:30.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:26:30.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:26:30.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:26:30.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:26:30.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:26:30.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:26:30.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:26:30.480 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-20T21:26:30.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:26:30.489 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:26:30.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=88114 2026-02-20T21:26:30.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 88114 ']' 2026-02-20T21:26:30.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill -KILL 88114 2026-02-20T21:26:30.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:26:30.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:26:30.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:26:30.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=88114 '$2 == pid {print; exit 1}' 2026-02-20T21:26:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:26:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:26:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=88114 '$2 == pid {print; exit 1}' 2026-02-20T21:26:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:26:30.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:26:30.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:26:30.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:26:30.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:26:30.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:26:30.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:26:30.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:26:30.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:26:30.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:26:30.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:26:30.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:26:30.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:26:30.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:26:30.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:26:30.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:26:30.557 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:26:30.557 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:26:30.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-20T21:26:30.559 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:26:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:26:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:26:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:26:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:26:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:26:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:26:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:26:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:26:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:26:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:26:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:26:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:26:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:26:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:26:30.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:26:30.570 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:26:30.572 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:2 2026-02-20T21:26:30.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:26:30.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:26:30.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:26:30.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:26:30.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:26:30.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:26:30.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:26:30.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:26:30.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:26:30.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:26:30.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:26:30.573 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:26:30.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:26:30.574 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-20T21:26:30.583 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:26:30.584 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:26:30.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:26:30.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:3 -KILL 2026-02-20T21:26:30.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:26:30.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig=-KILL 2026-02-20T21:26:30.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:26:30.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:26:30.586 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:3 2026-02-20T21:26:30.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:3 2026-02-20T21:26:30.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:26:30.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:26:30.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:26:30.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:26:30.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:26:30.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:26:30.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:26:30.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:26:30.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:26:30.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:26:30.587 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:26:30.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:26:30.588 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-20T21:26:30.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:26:30.597 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:26:30.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=88191 2026-02-20T21:26:30.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 88191 ']' 2026-02-20T21:26:30.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill -KILL 88191 2026-02-20T21:26:30.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:26:30.599 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:26:30.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:26:30.600 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=88191 '$2 == pid {print; exit 1}' 2026-02-20T21:26:30.626 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:26:30.627 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:26:30.627 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=88191 '$2 == pid {print; exit 1}' 2026-02-20T21:26:30.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:26:30.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:26:30.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:26:30.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:26:30.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:26:30.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:26:30.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:26:30.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:26:30.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:26:30.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:26:30.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:26:30.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:26:30.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:26:30.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:26:30.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:26:30.654 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:26:30.666 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:26:30.666 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:26:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-20T21:26:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:26:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:26:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:26:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:26:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:26:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:26:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:26:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:26:30.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:26:30.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:26:30.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:26:30.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:26:30.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:26:30.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:26:30.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:26:30.679 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:26:30.679 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:26:30.681 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:3 2026-02-20T21:26:30.681 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:26:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:26:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:26:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:26:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:26:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:26:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:26:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:26:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:26:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:26:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:26:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:26:30.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:26:30.683 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-20T21:26:30.692 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:26:30.692 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:26:30.693 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:26:30.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ get_snaps_json cluster1 mirror test_force_promote_user 2026-02-20T21:26:30.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:26:30.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local pool=mirror 2026-02-20T21:26:30.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local image=test_force_promote_user 2026-02-20T21:26:30.694 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 snap ls mirror/test_force_promote_user --all --format json 2026-02-20T21:26:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ SNAPS='[{"id":216,"name":".mirror.non_primary.632eca39-aab5-4396-b3b7-5b0733f0e4d3.5966961c-c4b1-43c9-b906-4ffa4d4d1d20","size":10737418240,"protected":"false","timestamp":"Fri Feb 20 21:26:18 2026","namespace":{"type":"mirror","state":"non-primary","mirror_peer_uuids":[],"complete":true,"primary_mirror_uuid":"2b58eacb-15a2-47fd-b97e-915fa36056e8","primary_snap_id":204,"last_copied_object_number":2560}},{"id":217,"name":"snap1","size":10737418240,"protected":"false","timestamp":"Fri Feb 20 21:26:27 2026","namespace":{"type":"user"}},{"id":218,"name":".mirror.non_primary.632eca39-aab5-4396-b3b7-5b0733f0e4d3.12754823-0d3f-4afb-9737-b57a0ca15dc8","size":10737418240,"protected":"false","timestamp":"Fri Feb 20 21:26:28 2026","namespace":{"type":"mirror","state":"non-primary","mirror_peer_uuids":[],"complete":false,"primary_mirror_uuid":"2b58eacb-15a2-47fd-b97e-915fa36056e8","primary_snap_id":206,"last_copied_object_number":972}}]' 2026-02-20T21:26:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ jq -e '.[-1].namespace["type"] == "mirror" and .[-1].namespace["state"] == "non-primary" and .[-1].namespace["complete"] == false' 2026-02-20T21:26:30.726 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:true 2026-02-20T21:26:30.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ promote_image cluster1 mirror test_force_promote_user --force 2026-02-20T21:26:30.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:26:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:26:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:26:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local force=--force 2026-02-20T21:26:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_force_promote_user --force 2026-02-20T21:27:06.003 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:Image promoted to primary 2026-02-20T21:27:06.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirrors cluster1 2026-02-20T21:27:06.010 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:27:06.011 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:27:06.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:06.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:0 2026-02-20T21:27:06.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:06.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:27:06.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:06.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:0 2026-02-20T21:27:06.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:06.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:27:06.013 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:27:06.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=0 2026-02-20T21:27:06.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:06.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:27:06.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:06.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=0 2026-02-20T21:27:06.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:27:06.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:27:06.014 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:27:06.033 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:1 2026-02-20T21:27:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:27:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:1 2026-02-20T21:27:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:27:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:27:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=1 2026-02-20T21:27:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:27:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=1 2026-02-20T21:27:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:27:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:27:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:27:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:2 2026-02-20T21:27:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:27:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:2 2026-02-20T21:27:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:27:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:27:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=2 2026-02-20T21:27:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:27:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=2 2026-02-20T21:27:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:27:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:27:06.057 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:27:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ start_mirror cluster1:3 2026-02-20T21:27:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:27:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance 2026-02-20T21:27:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:27:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local val=cluster1:3 2026-02-20T21:27:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local instance_var_name=instance 2026-02-20T21:27:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cluster=cluster1 2026-02-20T21:27:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ instance=3 2026-02-20T21:27:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval cluster=cluster1 2026-02-20T21:27:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:06.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ eval instance=3 2026-02-20T21:27:06.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:27:06.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:27:06.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-20T21:27:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster1 mirror test_force_promote_user 2026-02-20T21:27:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:27:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:27:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:27:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster1 mirror test_force_promote_user stopped 2026-02-20T21:27:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:27:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:27:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:27:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:27:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:27:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:27:07.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster1 mirror test_force_promote_user stopped 2026-02-20T21:27:07.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:27:07.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:27:07.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:27:07.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:27:07.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:27:07.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:27:07.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:07.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:27:07.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster1 2026-02-20T21:27:07.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:07.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:27:07.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster1 2026-02-20T21:27:07.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:27:07.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:07.099 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:27:07.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:07.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:27:07.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:07.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:27:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:27:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:27:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:27:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:27:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:27:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:27:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:27:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:07.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:07.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:27:07.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:27:07.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:07.113 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:07.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:07.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:07.114 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:07.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:07.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:07.154 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:27:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:27:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:27:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:27:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:27:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:27:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:27:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:27:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:07.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:27:07.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:27:07.160 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:07.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:07.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:07.167 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:07.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:07.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:07.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:27:07.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:07.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:07.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:27:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:27:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:27:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:27:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:27:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:27:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:27:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:27:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:27:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:07.223 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:07.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:07.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:07.224 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:07.264 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:07.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:07.265 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:27:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:27:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:27:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:27:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:27:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:27:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:27:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:27:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:07.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:07.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:27:07.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:27:07.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:07.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:07.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:07.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:27:07.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:07.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:07.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:07.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:27:07.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:27:07.271 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:27:07.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:07.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:07.278 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:07.279 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:07.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:07.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:07.319 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:07.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:07.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 1 2026-02-20T21:27:08.321 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:08.322 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:27:08.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:08.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:08.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:27:08.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:08.325 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:08.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:08.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:27:08.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:08.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:08.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:08.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:27:08.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:08.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:08.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:08.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:27:08.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:27:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:27:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:27:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:27:08.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:08.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:08.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:08.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:27:08.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:27:08.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:08.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:08.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:08.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:08.338 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:08.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:08.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:08.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:27:08.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:27:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:27:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:08.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:08.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:27:08.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:27:08.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:08.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:27:08.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:08.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:08.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:08.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:27:08.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:08.385 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:27:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:27:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:27:08.386 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:08.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:08.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:08.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:08.393 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:08.436 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:27:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:27:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:27:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:08.437 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:27:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:27:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:27:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:27:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:27:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:27:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:27:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:08.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:08.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:08.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:08.447 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:08.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:08.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:08.488 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:08.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:08.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:08.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:27:08.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:08.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:08.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:27:08.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:27:08.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:08.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:08.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:08.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:27:08.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:08.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:08.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:08.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:27:08.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:27:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:27:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:27:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:27:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:27:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:27:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:27:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:27:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:27:08.501 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:08.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:08.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:08.502 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:08.544 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:08.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:08.545 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 2 2026-02-20T21:27:10.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:10.547 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:27:10.550 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:10.550 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:27:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:27:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:27:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:27:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:27:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:27:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:27:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:10.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:27:10.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:10.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:10.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:10.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:27:10.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:27:10.553 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:10.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:10.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:10.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:10.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:10.607 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:10.607 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:10.607 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:27:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:27:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:27:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:27:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:27:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:27:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:27:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:27:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:10.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:10.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:10.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:27:10.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:27:10.610 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:10.617 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:10.618 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:10.618 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:10.618 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:10.657 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:10.657 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:10.657 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:27:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:27:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:27:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:27:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:27:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:27:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:27:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:27:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:27:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:27:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:10.670 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:10.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:10.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:10.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:10.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:10.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:10.711 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:10.714 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:10.714 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:10.714 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:27:10.714 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:10.714 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:10.714 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:27:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:27:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:27:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:27:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:27:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:27:10.715 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:27:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:27:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:27:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:27:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:10.717 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:27:10.717 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:27:10.717 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:27:10.724 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:10.724 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:10.724 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:10.724 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:10.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:10.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:10.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:10.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:10.766 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 4 2026-02-20T21:27:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:14.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:27:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:27:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:27:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:14.772 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:27:14.772 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:14.772 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:14.772 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:14.772 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:27:14.772 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:27:14.772 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:14.772 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:27:14.772 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:14.772 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:14.772 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:14.772 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:27:14.772 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:14.773 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:14.773 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:14.773 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:27:14.773 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:14.773 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:14.773 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:14.773 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:27:14.773 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:27:14.773 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:14.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:14.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:14.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:14.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:14.824 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:14.824 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:14.824 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:14.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:14.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:14.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:27:14.827 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:14.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:14.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:14.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:27:14.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:14.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:14.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:14.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:27:14.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:14.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:14.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:14.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:27:14.828 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:27:14.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:14.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:27:14.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:14.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:14.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:14.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:27:14.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:14.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:14.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:14.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:27:14.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:14.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:14.829 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:14.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:27:14.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:27:14.830 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:14.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:14.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:14.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:14.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:14.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:14.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:14.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:14.882 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:14.882 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:14.882 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:27:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:27:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:27:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:27:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:27:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:27:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:27:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:27:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:27:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:27:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:14.892 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:14.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:14.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:14.933 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:14.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:14.934 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:27:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:27:14.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:14.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:14.946 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:14.947 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:14.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:14.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:14.986 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:14.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:14.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 8 2026-02-20T21:27:22.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:22.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:27:22.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:22.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:22.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:27:22.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:22.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:22.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:22.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:27:22.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:27:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:27:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:27:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:27:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:22.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:22.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:22.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:27:22.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:22.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:22.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:22.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:27:22.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:22.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:22.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:22.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:27:22.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:27:22.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:23.005 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:23.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:23.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:23.006 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:23.049 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:23.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:23.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:27:23.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:23.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:23.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:23.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:27:23.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:23.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:23.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:23.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:27:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:27:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:27:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:27:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:27:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:27:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:27:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:27:23.052 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:23.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:23.060 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:23.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:23.061 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:23.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:23.102 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:23.103 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:27:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:27:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:27:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:27:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:27:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:27:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:27:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:27:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:27:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:27:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:23.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:23.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:23.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:23.117 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:23.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:23.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:27:23.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:23.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:23.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:27:23.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:27:23.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:23.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:23.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:23.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:27:23.163 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:23.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:23.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:23.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:27:23.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:27:23.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:27:23.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:27:23.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:23.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:23.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:27:23.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:27:23.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:23.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:23.164 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:23.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:27:23.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:23.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:23.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:23.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:27:23.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:27:23.165 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:27:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:23.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:23.173 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:23.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:23.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:23.213 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:23.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:23.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 8 2026-02-20T21:27:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:27:31.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:31.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:31.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:27:31.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:31.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:31.219 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:31.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:27:31.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:31.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:31.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:31.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:27:31.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:31.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:31.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:31.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:27:31.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:27:31.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:31.220 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:27:31.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:31.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:31.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:31.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:27:31.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:31.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:31.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:31.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:27:31.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:31.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:31.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:31.221 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:27:31.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:27:31.222 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:31.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:31.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:31.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:31.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:31.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:31.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:31.270 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:27:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:27:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:27:31.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:31.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:31.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:31.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:27:31.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:27:31.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:31.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:27:31.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:31.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:31.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:31.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:27:31.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:31.274 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:31.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:31.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:27:31.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:31.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:31.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:31.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:27:31.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:27:31.275 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:31.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:31.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:31.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:31.284 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:27:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:27:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:27:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:27:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:27:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:27:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:27:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:27:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:27:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:27:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:31.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:31.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:31.378 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:31.381 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:31.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:31.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:27:31.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:31.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:31.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:27:31.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:27:31.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:31.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:31.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:31.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:27:31.382 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:27:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:27:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:27:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:27:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:27:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:27:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:31.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:27:31.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:31.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:31.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:31.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:27:31.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:27:31.384 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:27:31.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:31.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:31.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:31.392 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:31.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:31.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:31.432 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:31.433 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:31.433 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 8 2026-02-20T21:27:39.434 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:39.435 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:27:39.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:39.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:39.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:27:39.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:39.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:39.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:39.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:27:39.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:39.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:39.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:39.438 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:27:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:27:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:27:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:27:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:27:39.440 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:39.440 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:39.440 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:39.440 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:27:39.440 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:39.440 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:39.440 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:39.440 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:27:39.440 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:27:39.440 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:39.449 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:39.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:39.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:39.450 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:39.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:39.491 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:39.492 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:39.494 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:39.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:39.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:27:39.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:39.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:39.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:39.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:27:39.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:39.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:39.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:39.495 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:27:39.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:39.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:39.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:39.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:27:39.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:27:39.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:39.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:27:39.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:39.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:39.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:39.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:27:39.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:39.496 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:39.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:39.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:27:39.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:39.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:39.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:39.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:27:39.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:27:39.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:39.504 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:39.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:39.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:39.505 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:39.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:39.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:39.546 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:39.549 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:39.549 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:39.550 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:27:39.550 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:39.550 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:39.550 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:39.550 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:27:39.550 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:39.550 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:39.550 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:39.550 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:27:39.550 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:39.550 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:39.550 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:39.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:27:39.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:27:39.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:39.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:27:39.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:39.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:39.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:39.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:27:39.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:39.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:39.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:39.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:27:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:27:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:27:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:39.601 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:39.601 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:39.601 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:39.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:39.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:39.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:27:39.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:39.604 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:39.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:27:39.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:27:39.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:39.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:39.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:39.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:27:39.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:39.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:39.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:39.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:27:39.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:27:39.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:27:39.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:27:39.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:39.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:39.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:27:39.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:27:39.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:39.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:39.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:39.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:27:39.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:39.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:39.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:39.607 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:27:39.607 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:27:39.607 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:27:39.614 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:39.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:39.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:39.615 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 8 2026-02-20T21:27:47.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:47.658 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:27:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:27:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:47.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:47.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:27:47.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:47.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:47.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:27:47.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:27:47.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:47.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:27:47.663 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:47.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:27:47.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:27:47.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:47.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:27:47.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1 2026-02-20T21:27:47.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:47.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:47.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:27:47.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:47.664 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:47.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:27:47.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:27:47.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:47.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:47.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:47.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:27:47.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:27:47.665 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:47.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:47.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:47.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:47.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:47.717 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:27:47.718 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:27:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:27:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:27:47.720 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:27:47.720 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:27:47.720 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_force_promote_user 2026-02-20T21:27:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:27:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:27:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:27:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_image_replay_state cluster2 mirror test_force_promote_user stopped 2026-02-20T21:27:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:27:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:27:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:27:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state=stopped 2026-02-20T21:27:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local s 2026-02-20T21:27:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:27:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_image_replay_state cluster2 mirror test_force_promote_user stopped 2026-02-20T21:27:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:27:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:27:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:27:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local test_state=stopped 2026-02-20T21:27:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status_result 2026-02-20T21:27:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local current_state=stopped 2026-02-20T21:27:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_instance=cluster2 2026-02-20T21:27:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:27:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance=cluster2 2026-02-20T21:27:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local loop_instance 2026-02-20T21:27:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sleep 0 2026-02-20T21:27:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ grep -i state 2026-02-20T21:27:48.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-20T21:27:48.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ seq 0 3 2026-02-20T21:27:48.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:48.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:48.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:27:48.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:48.727 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:48.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:27:48.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:27:48.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:48.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:48.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:27:48.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:27:48.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:27:48.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:27:48.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:27:48.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:27:48.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:27:48.728 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:27:48.729 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:0 2026-02-20T21:27:48.729 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:27:48.729 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:48.729 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:27:48.729 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:27:48.729 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:48.729 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:48.729 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:27:48.729 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:27:48.729 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:27:48.729 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:27:48.730 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:27:48.730 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:27:48.730 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:27:48.730 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:27:48.738 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:27:48.738 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:27:48.738 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:27:48.738 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:48.779 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-20T21:27:48.779 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:48.779 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:admin_socket: invalid command 2026-02-20T21:27:48.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:48.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:48.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:27:48.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ shift 2026-02-20T21:27:48.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:27:48.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:27:48.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:27:48.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:48.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:27:48.783 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:27:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:27:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:27:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:27:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:27:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:27:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:27:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:27:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local local_cluster=cluster2:1 2026-02-20T21:27:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2 2026-02-20T21:27:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:27:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:27:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:27:48.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:48.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:48.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:27:48.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:27:48.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:27:48.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval local_cluster=cluster2 2026-02-20T21:27:48.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ local_cluster=cluster2 2026-02-20T21:27:48.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:27:48.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:27:48.785 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:27:48.792 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:27:48.792 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:27:48.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:27:48.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-20T21:27:48.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ return 0 2026-02-20T21:27:48.838 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status_result=' "state": "Stopped"' 2026-02-20T21:27:48.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo ' "state": "Stopped"' 2026-02-20T21:27:48.839 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep -i Replaying 2026-02-20T21:27:48.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test stopped = stopped 2026-02-20T21:27:48.840 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:27:48.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_force_promote_user up+stopped 2026-02-20T21:27:48.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:27:48.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:27:48.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:27:48.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:27:48.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:27:48.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:27:48.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:48.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:27:49.841 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster1 mirror test_force_promote_user up+stopped '' '' 2026-02-20T21:27:49.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:27:49.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:27:49.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:27:49.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:27:49.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:27:49.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:27:49.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:27:49.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:27:49.842 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_force_promote_user 2026-02-20T21:27:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_force_promote_user: 2026-02-20T21:27:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 632eca39-aab5-4396-b3b7-5b0733f0e4d3 2026-02-20T21:27:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:27:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:27:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.0 on trial199 2026-02-20T21:27:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:27:46 2026-02-20T21:27:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:27:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster2-DNE 2026-02-20T21:27:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:27:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:27:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:27:49 2026-02-20T21:27:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:27:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 220 .mirror.primary.632eca39-aab5-4396-b3b7-5b0733f0e4d3.127bb2c4-1f7f-49be-b561-7516505dff2c (peer_uuids:[051657c0-214b-4bab-85b7-d4f8366fff9d])' 2026-02-20T21:27:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:27:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:27:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:27:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:27:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:27:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:27:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:27:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:27:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:27:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.0 on trial199 2026-02-20T21:27:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:27:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:27:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:27:46' 2026-02-20T21:27:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_force_promote_user up+stopped '2026-02-20 21:27:46' 2026-02-20T21:27:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:27:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:27:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:27:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:27:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:27:46' 2026-02-20T21:27:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:27:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:27:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:27:49.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK164046408mirror.0trial19920.2.0-677-gf7870454falseOK64106415mirror.1trial19920.2.0-677-gf7870454trueOK64176423mirror.2trial19920.2.0-677-gf7870454falseOK64306433mirror.3trial19920.2.0-677-gf7870454falseOKtest_force_promote_user632eca39-aab5-4396-b3b7-5b0733f0e4d3up+stoppedlocal image is primary64046408mirror.0trial1992026-02-20 21:27:46cluster2-DNE2b58eacb-15a2-47fd-b97e-915fa36056e8up+stoppedlocal image is primary2026-02-20 21:27:49' 2026-02-20T21:27:49.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:27:49.914 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/last_update' 2026-02-20T21:27:49.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:27:46' 2026-02-20T21:27:49.915 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/state' 2026-02-20T21:27:49.916 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:27:49.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:27:49.917 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:27:49.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:27:49.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:27:49.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:27:49.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_force_promote_user up+stopped 2026-02-20T21:27:49.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:27:49.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:27:49.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:27:49.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:27:49.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:27:49.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:27:49.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:49.918 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 1 2026-02-20T21:27:50.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_status_in_pool_dir cluster2 mirror test_force_promote_user up+stopped '' '' 2026-02-20T21:27:50.920 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:27:50.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:27:50.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:27:50.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:27:50.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local description_pattern= 2026-02-20T21:27:50.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local service_pattern= 2026-02-20T21:27:50.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:27:50.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:27:50.921 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_force_promote_user 2026-02-20T21:27:50.951 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='test_force_promote_user: 2026-02-20T21:27:50.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: global_id: 632eca39-aab5-4396-b3b7-5b0733f0e4d3 2026-02-20T21:27:50.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:27:50.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:27:50.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: service: mirror.1 on trial199 2026-02-20T21:27:50.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:27:49 2026-02-20T21:27:50.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: peer_sites: 2026-02-20T21:27:50.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: name: cluster1-DNE 2026-02-20T21:27:50.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: state: up+stopped 2026-02-20T21:27:50.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: description: local image is primary 2026-02-20T21:27:50.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: last_update: 2026-02-20 21:27:46 2026-02-20T21:27:50.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: snapshots: 2026-02-20T21:27:50.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 204 .mirror.primary.632eca39-aab5-4396-b3b7-5b0733f0e4d3.7a67a5d0-629a-41e5-a8f8-0ae0a0b0987b (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99]) 2026-02-20T21:27:50.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: 206 .mirror.primary.632eca39-aab5-4396-b3b7-5b0733f0e4d3.ac4c7c82-dce7-4d22-b11c-021add471109 (peer_uuids:[8f9db7bf-ab51-45f3-a841-1009ca678b99])' 2026-02-20T21:27:50.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ state: .*up+stopped' 2026-02-20T21:27:50.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: state: up+stopped 2026-02-20T21:27:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^ description: .*' 2026-02-20T21:27:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: description: local image is primary 2026-02-20T21:27:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n '' ']' 2026-02-20T21:27:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:27:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep '^up+' 2026-02-20T21:27:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:27:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:27:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: service: mirror.1 on trial199 2026-02-20T21:27:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update 2026-02-20T21:27:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:27:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:27:49' 2026-02-20T21:27:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_force_promote_user up+stopped '2026-02-20 21:27:49' 2026-02-20T21:27:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:27:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:27:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:27:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local state_pattern=up+stopped 2026-02-20T21:27:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local 'prev_last_update=2026-02-20 21:27:49' 2026-02-20T21:27:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local status 2026-02-20T21:27:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ CEPH_ARGS= 2026-02-20T21:27:50.959 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-20T21:27:50.987 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ status='OKOKOK163286331mirror.0trial19920.2.0-677-gf7870454trueOK63356338mirror.1trial19920.2.0-677-gf7870454falseOK63426350mirror.2trial19920.2.0-677-gf7870454falseOK63526358mirror.3trial19920.2.0-677-gf7870454falseOKtest_force_promote_user632eca39-aab5-4396-b3b7-5b0733f0e4d3up+stoppedlocal image is primary63356338mirror.1trial1992026-02-20 21:27:49cluster1-DNE16a05c91-e148-4fb4-a0b3-60f25e6947ebup+stoppedlocal image is primary2026-02-20 21:27:46' 2026-02-20T21:27:50.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local last_update state 2026-02-20T21:27:50.988 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/last_update' 2026-02-20T21:27:50.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ last_update='2026-02-20 21:27:49' 2026-02-20T21:27:50.989 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/state' 2026-02-20T21:27:50.990 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ state=up+stopped 2026-02-20T21:27:50.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo up+stopped 2026-02-20T21:27:50.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ grep up+stopped 2026-02-20T21:27:50.991 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:up+stopped 2026-02-20T21:27:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:27:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:27:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster1 mirror test_force_promote_user 100 2026-02-20T21:27:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:27:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:27:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:27:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:27:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:27:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:27:50.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:27:50.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster1 bench mirror/test_force_promote_user --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:27:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:27:51.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:27:51.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 2040.85 bytes/sec: 8.0 MiB/s 2026-02-20T21:27:51.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ write_image cluster2 mirror test_force_promote_user 100 2026-02-20T21:27:51.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:27:51.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:27:51.078 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:27:51.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local count=100 2026-02-20T21:27:51.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local size= 2026-02-20T21:27:51.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:27:51.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ size=4096 2026-02-20T21:27:51.079 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote_user --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-20T21:27:51.109 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-20T21:27:51.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-20T21:27:51.177 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:elapsed: 0 ops: 100 ops/sec: 1492.56 bytes/sec: 5.8 MiB/s 2026-02-20T21:27:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster1 mirror test_force_promote_user 2026-02-20T21:27:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:27:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:27:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:27:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:27:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:27:51.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster1 mirror test_force_promote_user 2026-02-20T21:27:51.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:27:51.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:27:51.184 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:27:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_force_promote_user 2026-02-20T21:27:52.032 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-02-20T21:27:52.043 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster1 rm mirror/test_force_promote_user 2026-02-20T21:27:53.073 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T21:27:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:27:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image_retry cluster2 mirror test_force_promote_user 2026-02-20T21:27:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:27:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:27:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:27:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:27:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:27:53.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ remove_image cluster2 mirror test_force_promote_user 2026-02-20T21:27:53.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:27:53.076 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pool=mirror 2026-02-20T21:27:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local image=test_force_promote_user 2026-02-20T21:27:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_force_promote_user 2026-02-20T21:27:54.050 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-02-20T21:27:54.058 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rbd --cluster=cluster2 rm mirror/test_force_promote_user 2026-02-20T21:27:56.453 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T21:27:56.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ return 0 2026-02-20T21:27:56.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ cleanup 0 2026-02-20T21:27:56.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local error_code=0 2026-02-20T21:27:56.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ set +e 2026-02-20T21:27:56.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 -ne 0 ']' 2026-02-20T21:27:56.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -z '' ']' 2026-02-20T21:27:56.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster instance 2026-02-20T21:27:56.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:27:56.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --cluster cluster1 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-20T21:27:57.097 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:pool 'mirror' does not exist 2026-02-20T21:27:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:27:57.105 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --cluster cluster2 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-20T21:27:58.124 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:pool 'mirror' does not exist 2026-02-20T21:27:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:27:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --cluster cluster1 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-20T21:27:59.098 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:pool 'mirror_parent' does not exist 2026-02-20T21:27:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ CEPH_ARGS= 2026-02-20T21:27:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ceph --cluster cluster2 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-20T21:28:00.119 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:pool 'mirror_parent' does not exist 2026-02-20T21:28:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-20T21:28:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirrors cluster1 2026-02-20T21:28:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1 2026-02-20T21:28:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:28:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:28:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:0 '' 2026-02-20T21:28:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:28:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:28:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:28:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:0 2026-02-20T21:28:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:0 2026-02-20T21:28:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:28:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:0 2026-02-20T21:28:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:28:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:28:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:28:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:28:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:28:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:28:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:28:00.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:28:00.133 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-20T21:28:00.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:28:00.144 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:28:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=89713 2026-02-20T21:28:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 89713 ']' 2026-02-20T21:28:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 89713 2026-02-20T21:28:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:28:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:28:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:28:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=89713 '$2 == pid {print; exit 1}' 2026-02-20T21:28:00.174 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:28:00.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:28:00.175 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=89713 '$2 == pid {print; exit 1}' 2026-02-20T21:28:00.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:00.201 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:28:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:28:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:28:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:28:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:28:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:28:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:00.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:00.214 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-20T21:28:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:28:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:28:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.216 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:28:00.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:28:00.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:00.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:00.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:00.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:28:00.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:28:00.217 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:00.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:00.227 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:0 2026-02-20T21:28:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:0 2026-02-20T21:28:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:0 2026-02-20T21:28:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:28:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:28:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:28:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:28:00.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:28:00.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:28:00.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:28:00.230 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-20T21:28:00.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:28:00.240 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-20T21:28:00.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:00.242 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:1 '' 2026-02-20T21:28:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:28:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:28:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:28:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:1 2026-02-20T21:28:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:1 2026-02-20T21:28:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:28:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:1 2026-02-20T21:28:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:28:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:28:00.243 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:28:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:28:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:28:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:28:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:28:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:28:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-20T21:28:00.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:28:00.254 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:28:00.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=89757 2026-02-20T21:28:00.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 89757 ']' 2026-02-20T21:28:00.255 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 89757 2026-02-20T21:28:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:28:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:28:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:28:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=89757 '$2 == pid {print; exit 1}' 2026-02-20T21:28:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:28:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:28:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=89757 '$2 == pid {print; exit 1}' 2026-02-20T21:28:00.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:00.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:00.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:28:00.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:00.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:28:00.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:00.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.309 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:28:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:28:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:28:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:28:00.310 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:00.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:00.320 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-20T21:28:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:28:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:28:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:28:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:28:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:00.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:00.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:00.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:28:00.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:28:00.324 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:00.333 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:00.334 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:1 2026-02-20T21:28:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:1 2026-02-20T21:28:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:1 2026-02-20T21:28:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:28:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:28:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:28:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:28:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:28:00.336 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:28:00.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:28:00.337 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-20T21:28:00.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:28:00.347 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-20T21:28:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:2 '' 2026-02-20T21:28:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:2 2026-02-20T21:28:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:28:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:28:00.349 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:28:00.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:2 2026-02-20T21:28:00.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:2 2026-02-20T21:28:00.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:28:00.350 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:28:00.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:2 2026-02-20T21:28:00.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:28:00.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:28:00.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:28:00.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:28:00.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:00.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:28:00.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:28:00.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:28:00.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:28:00.351 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-20T21:28:00.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:28:00.361 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:28:00.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=89812 2026-02-20T21:28:00.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 89812 ']' 2026-02-20T21:28:00.362 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 89812 2026-02-20T21:28:00.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:28:00.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:28:00.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:28:00.363 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=89812 '$2 == pid {print; exit 1}' 2026-02-20T21:28:00.390 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:28:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:28:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=89812 '$2 == pid {print; exit 1}' 2026-02-20T21:28:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:28:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:28:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:28:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:28:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:28:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:28:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:28:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:28:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:28:00.418 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:28:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:00.429 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-20T21:28:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:28:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:28:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:28:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:28:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:28:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:28:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:28:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:28:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:28:00.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:00.441 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:00.443 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:2 2026-02-20T21:28:00.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:2 2026-02-20T21:28:00.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:28:00.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:2 2026-02-20T21:28:00.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:28:00.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:28:00.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:28:00.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:00.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:28:00.444 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:28:00.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:28:00.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:28:00.445 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-20T21:28:00.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:28:00.455 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-20T21:28:00.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:00.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster1:3 '' 2026-02-20T21:28:00.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster1:3 2026-02-20T21:28:00.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:28:00.456 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:28:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:28:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster1:3 2026-02-20T21:28:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster1:3 2026-02-20T21:28:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:28:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:28:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster1:3 2026-02-20T21:28:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:28:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:28:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:28:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:28:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:28:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster1 2026-02-20T21:28:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster1 2026-02-20T21:28:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:28:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:28:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-20T21:28:00.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:28:00.469 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:28:00.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=89894 2026-02-20T21:28:00.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 89894 ']' 2026-02-20T21:28:00.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 89894 2026-02-20T21:28:00.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:28:00.471 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:28:00.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:28:00.472 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=89894 '$2 == pid {print; exit 1}' 2026-02-20T21:28:00.497 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:28:00.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:28:00.498 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=89894 '$2 == pid {print; exit 1}' 2026-02-20T21:28:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:28:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:28:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:28:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:28:00.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:28:00.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:00.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:28:00.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:28:00.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:28:00.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:00.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:00.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:28:00.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:28:00.524 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:28:00.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:28:00.535 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:28:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-20T21:28:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:28:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:28:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:28:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:28:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:28:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:28:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:28:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:28:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:28:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:28:00.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:28:00.548 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:28:00.550 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster1:3 2026-02-20T21:28:00.550 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1:3 2026-02-20T21:28:00.550 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.550 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:28:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster1:3 2026-02-20T21:28:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:28:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster1 2026-02-20T21:28:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:28:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:28:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster1 2026-02-20T21:28:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster1 2026-02-20T21:28:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:28:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:28:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-20T21:28:00.561 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:28:00.562 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-20T21:28:00.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-20T21:28:00.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirrors cluster2 2026-02-20T21:28:00.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2 2026-02-20T21:28:00.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:28:00.563 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ seq 0 3 2026-02-20T21:28:00.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:00.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster2:0 '' 2026-02-20T21:28:00.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2:0 2026-02-20T21:28:00.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:28:00.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:28:00.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:28:00.565 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster2:0 2026-02-20T21:28:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2:0 2026-02-20T21:28:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:28:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:28:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:0 2026-02-20T21:28:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:28:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:28:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:28:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:28:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-20T21:28:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster2 2026-02-20T21:28:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster2 2026-02-20T21:28:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=0 2026-02-20T21:28:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=0 2026-02-20T21:28:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-02-20T21:28:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-20T21:28:00.576 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-20T21:28:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=86080 2026-02-20T21:28:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 86080 ']' 2026-02-20T21:28:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 86080 2026-02-20T21:28:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:28:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:28:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:28:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=86080 '$2 == pid {print; exit 1}' 2026-02-20T21:28:00.605 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:28:00.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:28:00.606 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=86080 '$2 == pid {print; exit 1}' 2026-02-20T21:28:00.631 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster2:0 cluster1 2026-02-20T21:28:00.631 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster2:0 2026-02-20T21:28:00.631 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:28:00.631 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.631 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:28:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:28:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:28:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:28:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:28:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster2 2026-02-20T21:28:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster2 2026-02-20T21:28:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:28:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:28:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:28:00.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:28:00.643 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:28:00.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-20T21:28:00.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster2:0 2026-02-20T21:28:00.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:28:00.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.645 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-20T21:28:00.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:28:00.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:00.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:28:00.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:28:00.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:28:00.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster2 2026-02-20T21:28:00.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster2 2026-02-20T21:28:00.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:28:00.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:28:00.646 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-20T21:28:00.657 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:28:00.657 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-20T21:28:00.659 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster2:0 2026-02-20T21:28:00.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:0 2026-02-20T21:28:00.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:28:00.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:0 2026-02-20T21:28:00.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:28:00.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:28:00.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=0 2026-02-20T21:28:00.660 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-20T21:28:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:28:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:28:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=0 2026-02-20T21:28:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=0 2026-02-20T21:28:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-02-20T21:28:00.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-20T21:28:00.671 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-20T21:28:00.672 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:00.672 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster2:1 '' 2026-02-20T21:28:00.672 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2:1 2026-02-20T21:28:00.672 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:28:00.672 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:28:00.673 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:28:00.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster2:1 2026-02-20T21:28:00.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2:1 2026-02-20T21:28:00.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:28:00.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:28:00.674 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:1 2026-02-20T21:28:00.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:28:00.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:28:00.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:28:00.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:28:00.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-20T21:28:00.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster2 2026-02-20T21:28:00.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster2 2026-02-20T21:28:00.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=1 2026-02-20T21:28:00.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=1 2026-02-20T21:28:00.675 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'pid file' 2026-02-20T21:28:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-20T21:28:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-20T21:28:00.686 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=86124 2026-02-20T21:28:00.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 86124 ']' 2026-02-20T21:28:00.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 86124 2026-02-20T21:28:00.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:28:00.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:28:00.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:28:00.687 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=86124 '$2 == pid {print; exit 1}' 2026-02-20T21:28:00.714 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:28:00.714 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:28:00.714 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=86124 '$2 == pid {print; exit 1}' 2026-02-20T21:28:00.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster2:1 cluster1 2026-02-20T21:28:00.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster2:1 2026-02-20T21:28:00.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:28:00.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:28:00.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:28:00.740 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:00.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:28:00.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:28:00.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:28:00.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster2 2026-02-20T21:28:00.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster2 2026-02-20T21:28:00.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:28:00.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:28:00.741 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:28:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:28:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:28:00.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-20T21:28:00.753 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster2:1 2026-02-20T21:28:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:28:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-20T21:28:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:28:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:28:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:28:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:28:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster2 2026-02-20T21:28:00.754 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster2 2026-02-20T21:28:00.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:28:00.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:28:00.755 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-20T21:28:00.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:28:00.765 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-20T21:28:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster2:1 2026-02-20T21:28:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:1 2026-02-20T21:28:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:28:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:1 2026-02-20T21:28:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:28:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:28:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=1 2026-02-20T21:28:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-20T21:28:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:28:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:28:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=1 2026-02-20T21:28:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=1 2026-02-20T21:28:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'pid file' 2026-02-20T21:28:00.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-20T21:28:00.778 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-20T21:28:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster2:2 '' 2026-02-20T21:28:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2:2 2026-02-20T21:28:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:28:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:28:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:28:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster2:2 2026-02-20T21:28:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2:2 2026-02-20T21:28:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:28:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:2 cluster instance 2026-02-20T21:28:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:2 2026-02-20T21:28:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:28:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:28:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:28:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:28:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-20T21:28:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster2 2026-02-20T21:28:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster2 2026-02-20T21:28:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=2 2026-02-20T21:28:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=2 2026-02-20T21:28:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'pid file' 2026-02-20T21:28:00.791 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-20T21:28:00.792 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-20T21:28:00.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=86185 2026-02-20T21:28:00.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 86185 ']' 2026-02-20T21:28:00.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 86185 2026-02-20T21:28:00.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:28:00.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:28:00.793 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:28:00.794 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=86185 '$2 == pid {print; exit 1}' 2026-02-20T21:28:00.818 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:28:00.819 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:28:00.819 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=86185 '$2 == pid {print; exit 1}' 2026-02-20T21:28:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster2:2 cluster1 2026-02-20T21:28:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster2:2 2026-02-20T21:28:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:28:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-20T21:28:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:2 2026-02-20T21:28:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:28:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:28:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-20T21:28:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster2 2026-02-20T21:28:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster2 2026-02-20T21:28:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:28:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:28:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-20T21:28:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:28:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:28:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-20T21:28:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster2:2 2026-02-20T21:28:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:28:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-20T21:28:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:2 2026-02-20T21:28:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:28:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:28:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-20T21:28:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster2 2026-02-20T21:28:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster2 2026-02-20T21:28:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:28:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:28:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-20T21:28:00.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:28:00.870 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-20T21:28:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster2:2 2026-02-20T21:28:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:2 2026-02-20T21:28:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-20T21:28:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:2 2026-02-20T21:28:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:28:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:28:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=2 2026-02-20T21:28:00.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-20T21:28:00.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:28:00.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:28:00.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=2 2026-02-20T21:28:00.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=2 2026-02-20T21:28:00.873 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'pid file' 2026-02-20T21:28:00.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-20T21:28:00.884 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-20T21:28:00.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:00.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ stop_mirror cluster2:3 '' 2026-02-20T21:28:00.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local cluster=cluster2:3 2026-02-20T21:28:00.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local sig= 2026-02-20T21:28:00.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test -n '' 2026-02-20T21:28:00.886 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ local pid 2026-02-20T21:28:00.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ daemon_pid_file cluster2:3 2026-02-20T21:28:00.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster=cluster2:3 2026-02-20T21:28:00.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance 2026-02-20T21:28:00.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ set_cluster_instance cluster2:3 cluster instance 2026-02-20T21:28:00.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local val=cluster2:3 2026-02-20T21:28:00.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local cluster_var_name=cluster 2026-02-20T21:28:00.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local instance_var_name=instance 2026-02-20T21:28:00.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:28:00.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:28:00.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-20T21:28:00.887 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval cluster=cluster2 2026-02-20T21:28:00.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ cluster=cluster2 2026-02-20T21:28:00.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ eval instance=3 2026-02-20T21:28:00.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ instance=3 2026-02-20T21:28:00.888 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'pid file' 2026-02-20T21:28:00.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-20T21:28:00.897 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-20T21:28:00.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ pid=86262 2026-02-20T21:28:00.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -n 86262 ']' 2026-02-20T21:28:00.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ kill 86262 2026-02-20T21:28:00.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:28:00.898 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ sleep 0 2026-02-20T21:28:00.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:28:00.899 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=86262 '$2 == pid {print; exit 1}' 2026-02-20T21:28:00.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ break 2026-02-20T21:28:00.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ ps auxww 2026-02-20T21:28:00.927 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ awk -v pid=86262 '$2 == pid {print; exit 1}' 2026-02-20T21:28:00.952 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster2:3 cluster1 2026-02-20T21:28:00.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster2:3 2026-02-20T21:28:00.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster1 2026-02-20T21:28:00.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-20T21:28:00.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:3 2026-02-20T21:28:00.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:00.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:28:00.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:28:00.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-20T21:28:00.953 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster2 2026-02-20T21:28:00.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster2 2026-02-20T21:28:00.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:28:00.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:28:00.954 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-20T21:28:00.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:28:00.965 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:28:00.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-20T21:28:00.967 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local local_cluster=cluster2:3 2026-02-20T21:28:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2 2026-02-20T21:28:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-20T21:28:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:3 2026-02-20T21:28:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:28:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:28:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-20T21:28:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval local_cluster=cluster2 2026-02-20T21:28:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ local_cluster=cluster2 2026-02-20T21:28:00.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:28:00.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:28:00.969 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-20T21:28:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:28:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-20T21:28:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ daemon_pid_file cluster2:3 2026-02-20T21:28:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster=cluster2:3 2026-02-20T21:28:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance 2026-02-20T21:28:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-20T21:28:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local val=cluster2:3 2026-02-20T21:28:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local cluster_var_name=cluster 2026-02-20T21:28:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ cluster=cluster2 2026-02-20T21:28:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ instance=3 2026-02-20T21:28:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-20T21:28:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval cluster=cluster2 2026-02-20T21:28:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ cluster=cluster2 2026-02-20T21:28:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ eval instance=3 2026-02-20T21:28:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ instance=3 2026-02-20T21:28:00.982 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'pid file' 2026-02-20T21:28:00.992 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-20T21:28:00.993 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-20T21:28:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' -z 1 ']' 2026-02-20T21:28:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ test '' = /tmp/tmp.2FjbyoErqN 2026-02-20T21:28:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ rm -Rf /tmp/tmp.2FjbyoErqN 2026-02-20T21:28:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T21:28:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ echo OK 2026-02-20T21:28:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial199.stdout:OK 2026-02-20T21:28:00.996 INFO:tasks.workunit.cluster1.client.mirror.trial199.stderr:+ exit 0 2026-02-20T21:28:00.996 INFO:teuthology.orchestra.run:Running command with timeout 3600 2026-02-20T21:28:00.997 DEBUG:teuthology.orchestra.run.trial199:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp 2026-02-20T21:28:01.064 INFO:tasks.workunit:Stopping ['rbd/rbd_mirror.sh'] on cluster1.client.mirror... 2026-02-20T21:28:01.064 DEBUG:teuthology.orchestra.run.trial199:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.cluster1.client.mirror /home/ubuntu/cephtest/clone.cluster1.client.mirror 2026-02-20T21:28:01.426 DEBUG:teuthology.parallel:result is None 2026-02-20T21:28:01.427 DEBUG:teuthology.orchestra.run.trial199:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-20T21:28:01.451 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-20T21:28:01.452 DEBUG:teuthology.orchestra.run.trial199:> rmdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-20T21:28:01.508 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-20T21:28:01.508 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-20T21:28:01.575 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-20T21:28:01.575 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-20T21:28:01.754 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:28:01.754 INFO:teuthology.orchestra.run.trial199.stderr:dumped all 2026-02-20T21:28:01.763 INFO:teuthology.orchestra.run.trial199.stdout:{"pg_ready":true,"pg_map":{"version":1312,"stamp":"2026-02-20T21:28:00.056153+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":688684,"kb_used_data":210868,"kb_used_omap":6787,"kb_used_meta":470972,"kb_avail":282426836,"statfs":{"total":289910292480,"available":289205080064,"internally_reserved":0,"allocated":215928832,"data_stored":3234226673,"data_compressed":152095358,"data_compressed_allocated":201678848,"data_compressed_original":3226423296,"omap_allocated":6949999,"internal_metadata":482276241},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-7180172134,"num_objects":-1954,"num_object_clones":-198,"num_object_copies":-3908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1954,"num_whiteouts":0,"num_read":-139186,"num_read_kb":-16254842,"num_write":-31839,"num_write_kb":-25940331,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-3,"num_bytes_recovered":0,"num_keys_recovered":-3,"num_objects_omap":-29,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.009365"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1127,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.056777+0000","last_change":"2026-02-20T21:03:10.052984+0000","last_active":"2026-02-20T21:27:58.056777+0000","last_peered":"2026-02-20T21:27:58.056777+0000","last_clean":"2026-02-20T21:27:58.056777+0000","last_became_active":"2026-02-20T21:03:08.051994+0000","last_became_peered":"2026-02-20T21:03:08.051994+0000","last_unstale":"2026-02-20T21:27:58.056777+0000","last_undegraded":"2026-02-20T21:27:58.056777+0000","last_fullsized":"2026-02-20T21:27:58.056777+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:23:12.739248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061609399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1127,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.056433+0000","last_change":"2026-02-20T21:03:10.053169+0000","last_active":"2026-02-20T21:27:58.056433+0000","last_peered":"2026-02-20T21:27:58.056433+0000","last_clean":"2026-02-20T21:27:58.056433+0000","last_became_active":"2026-02-20T21:03:08.052053+0000","last_became_peered":"2026-02-20T21:03:08.052053+0000","last_unstale":"2026-02-20T21:27:58.056433+0000","last_undegraded":"2026-02-20T21:27:58.056433+0000","last_fullsized":"2026-02-20T21:27:58.056433+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:22:33.108819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079631999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1127,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.055409+0000","last_change":"2026-02-20T21:03:10.052860+0000","last_active":"2026-02-20T21:27:58.055409+0000","last_peered":"2026-02-20T21:27:58.055409+0000","last_clean":"2026-02-20T21:27:58.055409+0000","last_became_active":"2026-02-20T21:03:08.051990+0000","last_became_peered":"2026-02-20T21:03:08.051990+0000","last_unstale":"2026-02-20T21:27:58.055409+0000","last_undegraded":"2026-02-20T21:27:58.055409+0000","last_fullsized":"2026-02-20T21:27:58.055409+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:56:43.086257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050303499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1127,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.055068+0000","last_change":"2026-02-20T21:03:10.052617+0000","last_active":"2026-02-20T21:27:58.055068+0000","last_peered":"2026-02-20T21:27:58.055068+0000","last_clean":"2026-02-20T21:27:58.055068+0000","last_became_active":"2026-02-20T21:03:08.052009+0000","last_became_peered":"2026-02-20T21:03:08.052009+0000","last_unstale":"2026-02-20T21:27:58.055068+0000","last_undegraded":"2026-02-20T21:27:58.055068+0000","last_fullsized":"2026-02-20T21:27:58.055068+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:48:37.573173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039422300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":1127,"reported_epoch":563,"state":"active+clean","last_fresh":"2026-02-20T21:27:56.054413+0000","last_change":"2026-02-20T21:03:10.054541+0000","last_active":"2026-02-20T21:27:56.054413+0000","last_peered":"2026-02-20T21:27:56.054413+0000","last_clean":"2026-02-20T21:27:56.054413+0000","last_became_active":"2026-02-20T21:03:08.052246+0000","last_became_peered":"2026-02-20T21:03:08.052246+0000","last_unstale":"2026-02-20T21:27:56.054413+0000","last_undegraded":"2026-02-20T21:27:56.054413+0000","last_fullsized":"2026-02-20T21:27:56.054413+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:02:15.970404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00076615400000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1126,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.057600+0000","last_change":"2026-02-20T21:03:10.437050+0000","last_active":"2026-02-20T21:27:58.057600+0000","last_peered":"2026-02-20T21:27:58.057600+0000","last_clean":"2026-02-20T21:27:58.057600+0000","last_became_active":"2026-02-20T21:03:08.052284+0000","last_became_peered":"2026-02-20T21:03:08.052284+0000","last_unstale":"2026-02-20T21:27:58.057600+0000","last_undegraded":"2026-02-20T21:27:58.057600+0000","last_fullsized":"2026-02-20T21:27:58.057600+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:53:33.417610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030119000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1127,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.057259+0000","last_change":"2026-02-20T21:03:10.436978+0000","last_active":"2026-02-20T21:27:58.057259+0000","last_peered":"2026-02-20T21:27:58.057259+0000","last_clean":"2026-02-20T21:27:58.057259+0000","last_became_active":"2026-02-20T21:03:08.052106+0000","last_became_peered":"2026-02-20T21:03:08.052106+0000","last_unstale":"2026-02-20T21:27:58.057259+0000","last_undegraded":"2026-02-20T21:27:58.057259+0000","last_fullsized":"2026-02-20T21:27:58.057259+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:47:18.795582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000257268,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":1128,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.057826+0000","last_change":"2026-02-20T21:03:10.052664+0000","last_active":"2026-02-20T21:27:58.057826+0000","last_peered":"2026-02-20T21:27:58.057826+0000","last_clean":"2026-02-20T21:27:58.057826+0000","last_became_active":"2026-02-20T21:03:08.051931+0000","last_became_peered":"2026-02-20T21:03:08.051931+0000","last_unstale":"2026-02-20T21:27:58.057826+0000","last_undegraded":"2026-02-20T21:27:58.057826+0000","last_fullsized":"2026-02-20T21:27:58.057826+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:39:20.931975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000310737,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":1219,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.056821+0000","last_change":"2026-02-20T21:03:05.421656+0000","last_active":"2026-02-20T21:27:58.056821+0000","last_peered":"2026-02-20T21:27:58.056821+0000","last_clean":"2026-02-20T21:27:58.056821+0000","last_became_active":"2026-02-20T21:03:05.421208+0000","last_became_peered":"2026-02-20T21:03:05.421208+0000","last_unstale":"2026-02-20T21:27:58.056821+0000","last_undegraded":"2026-02-20T21:27:58.056821+0000","last_fullsized":"2026-02-20T21:27:58.056821+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:04.040085+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:04.040085+0000","last_clean_scrub_stamp":"2026-02-20T21:03:04.040085+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:35:31.987173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67134,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738670,"num_pgs":80,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":139708,"kb_used_data":4520,"kb_used_omap":2069,"kb_used_meta":133098,"kb_avail":94232132,"statfs":{"total":96636764160,"available":96493703168,"internally_reserved":0,"allocated":4628480,"data_stored":2481979,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2119148,"internal_metadata":136292884},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738671,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":193980,"kb_used_data":4584,"kb_used_omap":2120,"kb_used_meta":187255,"kb_avail":94177860,"statfs":{"total":96636764160,"available":96438128640,"internally_reserved":0,"allocated":4694016,"data_stored":3072347,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":2171022,"internal_metadata":191750002},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738671,"num_pgs":96,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":354996,"kb_used_data":201764,"kb_used_omap":2597,"kb_used_meta":150618,"kb_avail":94016844,"statfs":{"total":96636764160,"available":96273248256,"internally_reserved":0,"allocated":206606336,"data_stored":3228672347,"data_compressed":152061791,"data_compressed_allocated":201633792,"data_compressed_original":3225849856,"omap_allocated":2659829,"internal_metadata":154233355},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:28:01.764 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-20T21:28:01.942 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:28:01.943 INFO:teuthology.orchestra.run.trial199.stderr:dumped all 2026-02-20T21:28:01.950 INFO:teuthology.orchestra.run.trial199.stdout:{"pg_ready":true,"pg_map":{"version":1312,"stamp":"2026-02-20T21:28:00.056153+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":688684,"kb_used_data":210868,"kb_used_omap":6787,"kb_used_meta":470972,"kb_avail":282426836,"statfs":{"total":289910292480,"available":289205080064,"internally_reserved":0,"allocated":215928832,"data_stored":3234226673,"data_compressed":152095358,"data_compressed_allocated":201678848,"data_compressed_original":3226423296,"omap_allocated":6949999,"internal_metadata":482276241},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-7180172134,"num_objects":-1954,"num_object_clones":-198,"num_object_copies":-3908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1954,"num_whiteouts":0,"num_read":-139186,"num_read_kb":-16254842,"num_write":-31839,"num_write_kb":-25940331,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-3,"num_bytes_recovered":0,"num_keys_recovered":-3,"num_objects_omap":-29,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.009365"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1127,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.056777+0000","last_change":"2026-02-20T21:03:10.052984+0000","last_active":"2026-02-20T21:27:58.056777+0000","last_peered":"2026-02-20T21:27:58.056777+0000","last_clean":"2026-02-20T21:27:58.056777+0000","last_became_active":"2026-02-20T21:03:08.051994+0000","last_became_peered":"2026-02-20T21:03:08.051994+0000","last_unstale":"2026-02-20T21:27:58.056777+0000","last_undegraded":"2026-02-20T21:27:58.056777+0000","last_fullsized":"2026-02-20T21:27:58.056777+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:23:12.739248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061609399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1127,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.056433+0000","last_change":"2026-02-20T21:03:10.053169+0000","last_active":"2026-02-20T21:27:58.056433+0000","last_peered":"2026-02-20T21:27:58.056433+0000","last_clean":"2026-02-20T21:27:58.056433+0000","last_became_active":"2026-02-20T21:03:08.052053+0000","last_became_peered":"2026-02-20T21:03:08.052053+0000","last_unstale":"2026-02-20T21:27:58.056433+0000","last_undegraded":"2026-02-20T21:27:58.056433+0000","last_fullsized":"2026-02-20T21:27:58.056433+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:22:33.108819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079631999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1127,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.055409+0000","last_change":"2026-02-20T21:03:10.052860+0000","last_active":"2026-02-20T21:27:58.055409+0000","last_peered":"2026-02-20T21:27:58.055409+0000","last_clean":"2026-02-20T21:27:58.055409+0000","last_became_active":"2026-02-20T21:03:08.051990+0000","last_became_peered":"2026-02-20T21:03:08.051990+0000","last_unstale":"2026-02-20T21:27:58.055409+0000","last_undegraded":"2026-02-20T21:27:58.055409+0000","last_fullsized":"2026-02-20T21:27:58.055409+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:56:43.086257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050303499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1127,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.055068+0000","last_change":"2026-02-20T21:03:10.052617+0000","last_active":"2026-02-20T21:27:58.055068+0000","last_peered":"2026-02-20T21:27:58.055068+0000","last_clean":"2026-02-20T21:27:58.055068+0000","last_became_active":"2026-02-20T21:03:08.052009+0000","last_became_peered":"2026-02-20T21:03:08.052009+0000","last_unstale":"2026-02-20T21:27:58.055068+0000","last_undegraded":"2026-02-20T21:27:58.055068+0000","last_fullsized":"2026-02-20T21:27:58.055068+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:48:37.573173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039422300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":1127,"reported_epoch":563,"state":"active+clean","last_fresh":"2026-02-20T21:27:56.054413+0000","last_change":"2026-02-20T21:03:10.054541+0000","last_active":"2026-02-20T21:27:56.054413+0000","last_peered":"2026-02-20T21:27:56.054413+0000","last_clean":"2026-02-20T21:27:56.054413+0000","last_became_active":"2026-02-20T21:03:08.052246+0000","last_became_peered":"2026-02-20T21:03:08.052246+0000","last_unstale":"2026-02-20T21:27:56.054413+0000","last_undegraded":"2026-02-20T21:27:56.054413+0000","last_fullsized":"2026-02-20T21:27:56.054413+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:02:15.970404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00076615400000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1126,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.057600+0000","last_change":"2026-02-20T21:03:10.437050+0000","last_active":"2026-02-20T21:27:58.057600+0000","last_peered":"2026-02-20T21:27:58.057600+0000","last_clean":"2026-02-20T21:27:58.057600+0000","last_became_active":"2026-02-20T21:03:08.052284+0000","last_became_peered":"2026-02-20T21:03:08.052284+0000","last_unstale":"2026-02-20T21:27:58.057600+0000","last_undegraded":"2026-02-20T21:27:58.057600+0000","last_fullsized":"2026-02-20T21:27:58.057600+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:53:33.417610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030119000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1127,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.057259+0000","last_change":"2026-02-20T21:03:10.436978+0000","last_active":"2026-02-20T21:27:58.057259+0000","last_peered":"2026-02-20T21:27:58.057259+0000","last_clean":"2026-02-20T21:27:58.057259+0000","last_became_active":"2026-02-20T21:03:08.052106+0000","last_became_peered":"2026-02-20T21:03:08.052106+0000","last_unstale":"2026-02-20T21:27:58.057259+0000","last_undegraded":"2026-02-20T21:27:58.057259+0000","last_fullsized":"2026-02-20T21:27:58.057259+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:47:18.795582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000257268,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":1128,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.057826+0000","last_change":"2026-02-20T21:03:10.052664+0000","last_active":"2026-02-20T21:27:58.057826+0000","last_peered":"2026-02-20T21:27:58.057826+0000","last_clean":"2026-02-20T21:27:58.057826+0000","last_became_active":"2026-02-20T21:03:08.051931+0000","last_became_peered":"2026-02-20T21:03:08.051931+0000","last_unstale":"2026-02-20T21:27:58.057826+0000","last_undegraded":"2026-02-20T21:27:58.057826+0000","last_fullsized":"2026-02-20T21:27:58.057826+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:39:20.931975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000310737,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":1219,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.056821+0000","last_change":"2026-02-20T21:03:05.421656+0000","last_active":"2026-02-20T21:27:58.056821+0000","last_peered":"2026-02-20T21:27:58.056821+0000","last_clean":"2026-02-20T21:27:58.056821+0000","last_became_active":"2026-02-20T21:03:05.421208+0000","last_became_peered":"2026-02-20T21:03:05.421208+0000","last_unstale":"2026-02-20T21:27:58.056821+0000","last_undegraded":"2026-02-20T21:27:58.056821+0000","last_fullsized":"2026-02-20T21:27:58.056821+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:04.040085+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:04.040085+0000","last_clean_scrub_stamp":"2026-02-20T21:03:04.040085+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:35:31.987173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67134,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738670,"num_pgs":80,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":139708,"kb_used_data":4520,"kb_used_omap":2069,"kb_used_meta":133098,"kb_avail":94232132,"statfs":{"total":96636764160,"available":96493703168,"internally_reserved":0,"allocated":4628480,"data_stored":2481979,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2119148,"internal_metadata":136292884},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738671,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":193980,"kb_used_data":4584,"kb_used_omap":2120,"kb_used_meta":187255,"kb_avail":94177860,"statfs":{"total":96636764160,"available":96438128640,"internally_reserved":0,"allocated":4694016,"data_stored":3072347,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":2171022,"internal_metadata":191750002},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738671,"num_pgs":96,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":354996,"kb_used_data":201764,"kb_used_omap":2597,"kb_used_meta":150618,"kb_avail":94016844,"statfs":{"total":96636764160,"available":96273248256,"internally_reserved":0,"allocated":206606336,"data_stored":3228672347,"data_compressed":152061791,"data_compressed_allocated":201633792,"data_compressed_original":3225849856,"omap_allocated":2659829,"internal_metadata":154233355},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:28:01.952 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-20T21:28:01.952 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-20T21:28:02.224 INFO:teuthology.orchestra.run.trial199.stderr:2026-02-20T21:28:02.221+0000 7f7b1a7fc640 0 --2- 10.20.193.199:0/3144522074 >> v2:10.20.193.199:3300/0 conn(0x7f7b1c09bfe0 0x7f7b1c152c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2026-02-20T21:28:02.333 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:28:02.333 INFO:teuthology.orchestra.run.trial199.stderr:dumped all 2026-02-20T21:28:02.341 INFO:teuthology.orchestra.run.trial199.stdout:{"pg_ready":true,"pg_map":{"version":1313,"stamp":"2026-02-20T21:28:01.958506+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":688684,"kb_used_data":210868,"kb_used_omap":6787,"kb_used_meta":470972,"kb_avail":282426836,"statfs":{"total":289910292480,"available":289205080064,"internally_reserved":0,"allocated":215928832,"data_stored":3234226673,"data_compressed":152095358,"data_compressed_allocated":201678848,"data_compressed_original":3226423296,"omap_allocated":6949999,"internal_metadata":482276241},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-7180172134,"num_objects":-1954,"num_object_clones":-198,"num_object_copies":-3908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1954,"num_whiteouts":0,"num_read":-139186,"num_read_kb":-16254842,"num_write":-31839,"num_write_kb":-25940331,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-3,"num_bytes_recovered":0,"num_keys_recovered":-3,"num_objects_omap":-29,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.000824"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1127,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.056777+0000","last_change":"2026-02-20T21:03:10.052984+0000","last_active":"2026-02-20T21:27:58.056777+0000","last_peered":"2026-02-20T21:27:58.056777+0000","last_clean":"2026-02-20T21:27:58.056777+0000","last_became_active":"2026-02-20T21:03:08.051994+0000","last_became_peered":"2026-02-20T21:03:08.051994+0000","last_unstale":"2026-02-20T21:27:58.056777+0000","last_undegraded":"2026-02-20T21:27:58.056777+0000","last_fullsized":"2026-02-20T21:27:58.056777+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:23:12.739248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061609399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1127,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.056433+0000","last_change":"2026-02-20T21:03:10.053169+0000","last_active":"2026-02-20T21:27:58.056433+0000","last_peered":"2026-02-20T21:27:58.056433+0000","last_clean":"2026-02-20T21:27:58.056433+0000","last_became_active":"2026-02-20T21:03:08.052053+0000","last_became_peered":"2026-02-20T21:03:08.052053+0000","last_unstale":"2026-02-20T21:27:58.056433+0000","last_undegraded":"2026-02-20T21:27:58.056433+0000","last_fullsized":"2026-02-20T21:27:58.056433+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:22:33.108819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079631999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1127,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.055409+0000","last_change":"2026-02-20T21:03:10.052860+0000","last_active":"2026-02-20T21:27:58.055409+0000","last_peered":"2026-02-20T21:27:58.055409+0000","last_clean":"2026-02-20T21:27:58.055409+0000","last_became_active":"2026-02-20T21:03:08.051990+0000","last_became_peered":"2026-02-20T21:03:08.051990+0000","last_unstale":"2026-02-20T21:27:58.055409+0000","last_undegraded":"2026-02-20T21:27:58.055409+0000","last_fullsized":"2026-02-20T21:27:58.055409+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:56:43.086257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050303499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1127,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.055068+0000","last_change":"2026-02-20T21:03:10.052617+0000","last_active":"2026-02-20T21:27:58.055068+0000","last_peered":"2026-02-20T21:27:58.055068+0000","last_clean":"2026-02-20T21:27:58.055068+0000","last_became_active":"2026-02-20T21:03:08.052009+0000","last_became_peered":"2026-02-20T21:03:08.052009+0000","last_unstale":"2026-02-20T21:27:58.055068+0000","last_undegraded":"2026-02-20T21:27:58.055068+0000","last_fullsized":"2026-02-20T21:27:58.055068+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:48:37.573173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039422300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":1127,"reported_epoch":563,"state":"active+clean","last_fresh":"2026-02-20T21:27:56.054413+0000","last_change":"2026-02-20T21:03:10.054541+0000","last_active":"2026-02-20T21:27:56.054413+0000","last_peered":"2026-02-20T21:27:56.054413+0000","last_clean":"2026-02-20T21:27:56.054413+0000","last_became_active":"2026-02-20T21:03:08.052246+0000","last_became_peered":"2026-02-20T21:03:08.052246+0000","last_unstale":"2026-02-20T21:27:56.054413+0000","last_undegraded":"2026-02-20T21:27:56.054413+0000","last_fullsized":"2026-02-20T21:27:56.054413+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:02:15.970404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00076615400000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1126,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.057600+0000","last_change":"2026-02-20T21:03:10.437050+0000","last_active":"2026-02-20T21:27:58.057600+0000","last_peered":"2026-02-20T21:27:58.057600+0000","last_clean":"2026-02-20T21:27:58.057600+0000","last_became_active":"2026-02-20T21:03:08.052284+0000","last_became_peered":"2026-02-20T21:03:08.052284+0000","last_unstale":"2026-02-20T21:27:58.057600+0000","last_undegraded":"2026-02-20T21:27:58.057600+0000","last_fullsized":"2026-02-20T21:27:58.057600+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:53:33.417610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030119000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1127,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.057259+0000","last_change":"2026-02-20T21:03:10.436978+0000","last_active":"2026-02-20T21:27:58.057259+0000","last_peered":"2026-02-20T21:27:58.057259+0000","last_clean":"2026-02-20T21:27:58.057259+0000","last_became_active":"2026-02-20T21:03:08.052106+0000","last_became_peered":"2026-02-20T21:03:08.052106+0000","last_unstale":"2026-02-20T21:27:58.057259+0000","last_undegraded":"2026-02-20T21:27:58.057259+0000","last_fullsized":"2026-02-20T21:27:58.057259+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:47:18.795582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000257268,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":1128,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.057826+0000","last_change":"2026-02-20T21:03:10.052664+0000","last_active":"2026-02-20T21:27:58.057826+0000","last_peered":"2026-02-20T21:27:58.057826+0000","last_clean":"2026-02-20T21:27:58.057826+0000","last_became_active":"2026-02-20T21:03:08.051931+0000","last_became_peered":"2026-02-20T21:03:08.051931+0000","last_unstale":"2026-02-20T21:27:58.057826+0000","last_undegraded":"2026-02-20T21:27:58.057826+0000","last_fullsized":"2026-02-20T21:27:58.057826+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:39:20.931975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000310737,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":1219,"reported_epoch":564,"state":"active+clean","last_fresh":"2026-02-20T21:27:58.056821+0000","last_change":"2026-02-20T21:03:05.421656+0000","last_active":"2026-02-20T21:27:58.056821+0000","last_peered":"2026-02-20T21:27:58.056821+0000","last_clean":"2026-02-20T21:27:58.056821+0000","last_became_active":"2026-02-20T21:03:05.421208+0000","last_became_peered":"2026-02-20T21:03:05.421208+0000","last_unstale":"2026-02-20T21:27:58.056821+0000","last_undegraded":"2026-02-20T21:27:58.056821+0000","last_fullsized":"2026-02-20T21:27:58.056821+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:04.040085+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:04.040085+0000","last_clean_scrub_stamp":"2026-02-20T21:03:04.040085+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:35:31.987173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67134,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738670,"num_pgs":80,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":139708,"kb_used_data":4520,"kb_used_omap":2069,"kb_used_meta":133098,"kb_avail":94232132,"statfs":{"total":96636764160,"available":96493703168,"internally_reserved":0,"allocated":4628480,"data_stored":2481979,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2119148,"internal_metadata":136292884},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738671,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":193980,"kb_used_data":4584,"kb_used_omap":2120,"kb_used_meta":187255,"kb_avail":94177860,"statfs":{"total":96636764160,"available":96438128640,"internally_reserved":0,"allocated":4694016,"data_stored":3072347,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":2171022,"internal_metadata":191750002},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738671,"num_pgs":96,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":354996,"kb_used_data":201764,"kb_used_omap":2597,"kb_used_meta":150618,"kb_avail":94016844,"statfs":{"total":96636764160,"available":96273248256,"internally_reserved":0,"allocated":206606336,"data_stored":3228672347,"data_compressed":152061791,"data_compressed_allocated":201633792,"data_compressed_original":3225849856,"omap_allocated":2659829,"internal_metadata":154233355},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:28:02.343 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-20T21:28:02.525 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:28:02.525 INFO:teuthology.orchestra.run.trial199.stdout:{"epoch":565,"fsid":"f480090c-6109-4036-a7fa-8e7cbf4099eb","created":"2026-02-20T21:02:58.028370+0000","modified":"2026-02-20T21:28:00.053314+0000","last_up_change":"2026-02-20T21:03:02.035109+0000","last_in_change":"2026-02-20T21:02:58.696055+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:03:03.747219+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-20T21:03:06.093359+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ea3bd762-09f9-4bf7-b187-ff293d2e7ade","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":28,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6808","nonce":2744132852},{"type":"v1","addr":"10.20.193.199:6809","nonce":2744132852}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6810","nonce":2744132852},{"type":"v1","addr":"10.20.193.199:6811","nonce":2744132852}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6816","nonce":2744132852},{"type":"v1","addr":"10.20.193.199:6818","nonce":2744132852}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6812","nonce":2744132852},{"type":"v1","addr":"10.20.193.199:6814","nonce":2744132852}]},"public_addr":"10.20.193.199:6809/2744132852","cluster_addr":"10.20.193.199:6811/2744132852","heartbeat_back_addr":"10.20.193.199:6818/2744132852","heartbeat_front_addr":"10.20.193.199:6814/2744132852","state":["exists","up"]},{"osd":1,"uuid":"7b93c7a9-2b3d-4232-9811-774ad9478d1b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":28,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6800","nonce":2633081233},{"type":"v1","addr":"10.20.193.199:6801","nonce":2633081233}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6802","nonce":2633081233},{"type":"v1","addr":"10.20.193.199:6803","nonce":2633081233}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6806","nonce":2633081233},{"type":"v1","addr":"10.20.193.199:6807","nonce":2633081233}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6804","nonce":2633081233},{"type":"v1","addr":"10.20.193.199:6805","nonce":2633081233}]},"public_addr":"10.20.193.199:6801/2633081233","cluster_addr":"10.20.193.199:6803/2633081233","heartbeat_back_addr":"10.20.193.199:6807/2633081233","heartbeat_front_addr":"10.20.193.199:6805/2633081233","state":["exists","up"]},{"osd":2,"uuid":"13b69e07-8c48-41b2-bd6a-7ffe80cec20b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":28,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6813","nonce":3547377386},{"type":"v1","addr":"10.20.193.199:6815","nonce":3547377386}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6817","nonce":3547377386},{"type":"v1","addr":"10.20.193.199:6819","nonce":3547377386}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6822","nonce":3547377386},{"type":"v1","addr":"10.20.193.199:6823","nonce":3547377386}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.199:6820","nonce":3547377386},{"type":"v1","addr":"10.20.193.199:6821","nonce":3547377386}]},"public_addr":"10.20.193.199:6815/3547377386","cluster_addr":"10.20.193.199:6819/3547377386","heartbeat_back_addr":"10.20.193.199:6823/3547377386","heartbeat_front_addr":"10.20.193.199:6821/3547377386","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:03:00.564121+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:03:00.482246+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:03:00.469179+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":204,"length":3}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:28:03.536 INFO:tasks.ceph:Scrubbing cluster2.osd.0 2026-02-20T21:28:03.536 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-20T21:28:03.621 INFO:teuthology.orchestra.run.trial199.stdout:{ 2026-02-20T21:28:03.621 INFO:teuthology.orchestra.run.trial199.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-20T21:28:03.621 INFO:teuthology.orchestra.run.trial199.stdout:} 2026-02-20T21:28:03.628 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 0 2026-02-20T21:28:03.806 INFO:teuthology.orchestra.run.trial199.stderr:instructed osd(s) 0 to deep-scrub 2026-02-20T21:28:03.815 INFO:tasks.ceph:Scrubbing cluster2.osd.1 2026-02-20T21:28:03.816 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-20T21:28:03.898 INFO:teuthology.orchestra.run.trial199.stdout:{ 2026-02-20T21:28:03.899 INFO:teuthology.orchestra.run.trial199.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-20T21:28:03.899 INFO:teuthology.orchestra.run.trial199.stdout:} 2026-02-20T21:28:03.905 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 1 2026-02-20T21:28:04.084 INFO:teuthology.orchestra.run.trial199.stderr:instructed osd(s) 1 to deep-scrub 2026-02-20T21:28:04.094 INFO:tasks.ceph:Scrubbing cluster2.osd.2 2026-02-20T21:28:04.095 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-20T21:28:04.175 INFO:teuthology.orchestra.run.trial199.stdout:{ 2026-02-20T21:28:04.175 INFO:teuthology.orchestra.run.trial199.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-20T21:28:04.175 INFO:teuthology.orchestra.run.trial199.stdout:} 2026-02-20T21:28:04.182 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 2 2026-02-20T21:28:04.359 INFO:teuthology.orchestra.run.trial199.stderr:instructed osd(s) 2 to deep-scrub 2026-02-20T21:28:04.368 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-20T21:28:04.545 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:28:04.545 INFO:teuthology.orchestra.run.trial199.stderr:dumped all 2026-02-20T21:28:04.554 INFO:teuthology.orchestra.run.trial199.stdout:{"pg_ready":true,"pg_map":{"version":1314,"stamp":"2026-02-20T21:28:03.958672+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":99,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":509536,"kb_used_data":13744,"kb_used_omap":6792,"kb_used_meta":488951,"kb_avail":282605984,"statfs":{"total":289910292480,"available":289388527616,"internally_reserved":0,"allocated":14073856,"data_stored":8649988,"data_compressed":67134,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":6955764,"internal_metadata":500686092},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-7180172134,"num_objects":-1954,"num_object_clones":-198,"num_object_copies":-3908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1954,"num_whiteouts":0,"num_read":-139186,"num_read_kb":-16254842,"num_write":-31839,"num_write_kb":-25940331,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-3,"num_bytes_recovered":0,"num_keys_recovered":-3,"num_objects_omap":-29,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.909576"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1129,"reported_epoch":565,"state":"active+clean","last_fresh":"2026-02-20T21:28:00.149497+0000","last_change":"2026-02-20T21:03:10.052984+0000","last_active":"2026-02-20T21:28:00.149497+0000","last_peered":"2026-02-20T21:28:00.149497+0000","last_clean":"2026-02-20T21:28:00.149497+0000","last_became_active":"2026-02-20T21:03:08.051994+0000","last_became_peered":"2026-02-20T21:03:08.051994+0000","last_unstale":"2026-02-20T21:28:00.149497+0000","last_undegraded":"2026-02-20T21:28:00.149497+0000","last_fullsized":"2026-02-20T21:28:00.149497+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:23:12.739248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061609399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1129,"reported_epoch":565,"state":"active+clean","last_fresh":"2026-02-20T21:28:00.150206+0000","last_change":"2026-02-20T21:03:10.053169+0000","last_active":"2026-02-20T21:28:00.150206+0000","last_peered":"2026-02-20T21:28:00.150206+0000","last_clean":"2026-02-20T21:28:00.150206+0000","last_became_active":"2026-02-20T21:03:08.052053+0000","last_became_peered":"2026-02-20T21:03:08.052053+0000","last_unstale":"2026-02-20T21:28:00.150206+0000","last_undegraded":"2026-02-20T21:28:00.150206+0000","last_fullsized":"2026-02-20T21:28:00.150206+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:22:33.108819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079631999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1129,"reported_epoch":565,"state":"active+clean","last_fresh":"2026-02-20T21:28:00.147279+0000","last_change":"2026-02-20T21:03:10.052860+0000","last_active":"2026-02-20T21:28:00.147279+0000","last_peered":"2026-02-20T21:28:00.147279+0000","last_clean":"2026-02-20T21:28:00.147279+0000","last_became_active":"2026-02-20T21:03:08.051990+0000","last_became_peered":"2026-02-20T21:03:08.051990+0000","last_unstale":"2026-02-20T21:28:00.147279+0000","last_undegraded":"2026-02-20T21:28:00.147279+0000","last_fullsized":"2026-02-20T21:28:00.147279+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:56:43.086257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050303499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1129,"reported_epoch":565,"state":"active+clean","last_fresh":"2026-02-20T21:28:00.148978+0000","last_change":"2026-02-20T21:03:10.052617+0000","last_active":"2026-02-20T21:28:00.148978+0000","last_peered":"2026-02-20T21:28:00.148978+0000","last_clean":"2026-02-20T21:28:00.148978+0000","last_became_active":"2026-02-20T21:03:08.052009+0000","last_became_peered":"2026-02-20T21:03:08.052009+0000","last_unstale":"2026-02-20T21:28:00.148978+0000","last_undegraded":"2026-02-20T21:28:00.148978+0000","last_fullsized":"2026-02-20T21:28:00.148978+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:48:37.573173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039422300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":1131,"reported_epoch":565,"state":"active+clean","last_fresh":"2026-02-20T21:28:01.199791+0000","last_change":"2026-02-20T21:03:10.054541+0000","last_active":"2026-02-20T21:28:01.199791+0000","last_peered":"2026-02-20T21:28:01.199791+0000","last_clean":"2026-02-20T21:28:01.199791+0000","last_became_active":"2026-02-20T21:03:08.052246+0000","last_became_peered":"2026-02-20T21:03:08.052246+0000","last_unstale":"2026-02-20T21:28:01.199791+0000","last_undegraded":"2026-02-20T21:28:01.199791+0000","last_fullsized":"2026-02-20T21:28:01.199791+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:02:15.970404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00076615400000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1128,"reported_epoch":565,"state":"active+clean","last_fresh":"2026-02-20T21:28:00.059076+0000","last_change":"2026-02-20T21:03:10.437050+0000","last_active":"2026-02-20T21:28:00.059076+0000","last_peered":"2026-02-20T21:28:00.059076+0000","last_clean":"2026-02-20T21:28:00.059076+0000","last_became_active":"2026-02-20T21:03:08.052284+0000","last_became_peered":"2026-02-20T21:03:08.052284+0000","last_unstale":"2026-02-20T21:28:00.059076+0000","last_undegraded":"2026-02-20T21:28:00.059076+0000","last_fullsized":"2026-02-20T21:28:00.059076+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:53:33.417610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030119000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1129,"reported_epoch":565,"state":"active+clean","last_fresh":"2026-02-20T21:28:00.059589+0000","last_change":"2026-02-20T21:03:10.436978+0000","last_active":"2026-02-20T21:28:00.059589+0000","last_peered":"2026-02-20T21:28:00.059589+0000","last_clean":"2026-02-20T21:28:00.059589+0000","last_became_active":"2026-02-20T21:03:08.052106+0000","last_became_peered":"2026-02-20T21:03:08.052106+0000","last_unstale":"2026-02-20T21:28:00.059589+0000","last_undegraded":"2026-02-20T21:28:00.059589+0000","last_fullsized":"2026-02-20T21:28:00.059589+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:47:18.795582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000257268,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":1130,"reported_epoch":565,"state":"active+clean","last_fresh":"2026-02-20T21:28:00.148677+0000","last_change":"2026-02-20T21:03:10.052664+0000","last_active":"2026-02-20T21:28:00.148677+0000","last_peered":"2026-02-20T21:28:00.148677+0000","last_clean":"2026-02-20T21:28:00.148677+0000","last_became_active":"2026-02-20T21:03:08.051931+0000","last_became_peered":"2026-02-20T21:03:08.051931+0000","last_unstale":"2026-02-20T21:28:00.148677+0000","last_undegraded":"2026-02-20T21:28:00.148677+0000","last_fullsized":"2026-02-20T21:28:00.148677+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:07.044282+0000","last_clean_scrub_stamp":"2026-02-20T21:03:07.044282+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:39:20.931975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000310737,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":1221,"reported_epoch":565,"state":"active+clean","last_fresh":"2026-02-20T21:28:00.149630+0000","last_change":"2026-02-20T21:03:05.421656+0000","last_active":"2026-02-20T21:28:00.149630+0000","last_peered":"2026-02-20T21:28:00.149630+0000","last_clean":"2026-02-20T21:28:00.149630+0000","last_became_active":"2026-02-20T21:03:05.421208+0000","last_became_peered":"2026-02-20T21:03:05.421208+0000","last_unstale":"2026-02-20T21:28:00.149630+0000","last_undegraded":"2026-02-20T21:28:00.149630+0000","last_fullsized":"2026-02-20T21:28:00.149630+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:03:04.040085+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:03:04.040085+0000","last_clean_scrub_stamp":"2026-02-20T21:03:04.040085+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:35:31.987173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67134,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738671,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":139720,"kb_used_data":4536,"kb_used_omap":2073,"kb_used_meta":133094,"kb_avail":94232120,"statfs":{"total":96636764160,"available":96493690880,"internally_reserved":0,"allocated":4644864,"data_stored":2489738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2123596,"internal_metadata":136288436},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738672,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":193996,"kb_used_data":4604,"kb_used_omap":2115,"kb_used_meta":187260,"kb_avail":94177844,"statfs":{"total":96636764160,"available":96438112256,"internally_reserved":0,"allocated":4714496,"data_stored":3080125,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":2166315,"internal_metadata":191754709},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738672,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":175820,"kb_used_data":4604,"kb_used_omap":2603,"kb_used_meta":168596,"kb_avail":94196020,"statfs":{"total":96636764160,"available":96456724480,"internally_reserved":0,"allocated":4714496,"data_stored":3080125,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":2665853,"internal_metadata":172642947},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:28:04.559 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-20T21:03:07.044282+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=3, tm_sec=7, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=28, tm_sec=2, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:28:04.559 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-20T21:03:07.044282+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=3, tm_sec=7, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=28, tm_sec=2, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:28:04.560 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-20T21:03:07.044282+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=3, tm_sec=7, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=28, tm_sec=2, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:28:04.560 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-20T21:03:07.044282+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=3, tm_sec=7, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=28, tm_sec=2, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:28:04.560 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-20T21:03:07.044282+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=3, tm_sec=7, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=28, tm_sec=2, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:28:04.560 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-20T21:03:07.044282+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=3, tm_sec=7, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=28, tm_sec=2, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:28:04.560 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-20T21:03:07.044282+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=3, tm_sec=7, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=28, tm_sec=2, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:28:04.560 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-20T21:03:07.044282+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=3, tm_sec=7, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=28, tm_sec=2, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:28:04.561 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-20T21:03:04.040085+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=3, tm_sec=4, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=28, tm_sec=2, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:28:04.561 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-20T21:28:24.562 DEBUG:teuthology.orchestra.run.trial199:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-20T21:28:24.743 INFO:teuthology.orchestra.run.trial199.stdout: 2026-02-20T21:28:24.743 INFO:teuthology.orchestra.run.trial199.stderr:dumped all 2026-02-20T21:28:24.751 INFO:teuthology.orchestra.run.trial199.stdout:{"pg_ready":true,"pg_map":{"version":1324,"stamp":"2026-02-20T21:28:23.960182+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":509536,"kb_used_data":13744,"kb_used_omap":6792,"kb_used_meta":488951,"kb_avail":282605984,"statfs":{"total":289910292480,"available":289388527616,"internally_reserved":0,"allocated":14073856,"data_stored":8649988,"data_compressed":67134,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":6955764,"internal_metadata":500686092},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000919"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1137,"reported_epoch":565,"state":"active+clean","last_fresh":"2026-02-20T21:28:09.567314+0000","last_change":"2026-02-20T21:28:09.567314+0000","last_active":"2026-02-20T21:28:09.567314+0000","last_peered":"2026-02-20T21:28:09.567314+0000","last_clean":"2026-02-20T21:28:09.567314+0000","last_became_active":"2026-02-20T21:03:08.051994+0000","last_became_peered":"2026-02-20T21:03:08.051994+0000","last_unstale":"2026-02-20T21:28:09.567314+0000","last_undegraded":"2026-02-20T21:28:09.567314+0000","last_fullsized":"2026-02-20T21:28:09.567314+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:28:09.567246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:28:09.567246+0000","last_clean_scrub_stamp":"2026-02-20T21:28:09.567246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:59:57.230839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061609399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1137,"reported_epoch":565,"state":"active+clean","last_fresh":"2026-02-20T21:28:08.529580+0000","last_change":"2026-02-20T21:28:08.529580+0000","last_active":"2026-02-20T21:28:08.529580+0000","last_peered":"2026-02-20T21:28:08.529580+0000","last_clean":"2026-02-20T21:28:08.529580+0000","last_became_active":"2026-02-20T21:03:08.052053+0000","last_became_peered":"2026-02-20T21:03:08.052053+0000","last_unstale":"2026-02-20T21:28:08.529580+0000","last_undegraded":"2026-02-20T21:28:08.529580+0000","last_fullsized":"2026-02-20T21:28:08.529580+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:28:08.529483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:28:08.529483+0000","last_clean_scrub_stamp":"2026-02-20T21:28:08.529483+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:54:48.372763+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00079631999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1138,"reported_epoch":565,"state":"active+clean","last_fresh":"2026-02-20T21:28:06.451888+0000","last_change":"2026-02-20T21:28:06.451844+0000","last_active":"2026-02-20T21:28:06.451888+0000","last_peered":"2026-02-20T21:28:06.451888+0000","last_clean":"2026-02-20T21:28:06.451888+0000","last_became_active":"2026-02-20T21:03:08.051990+0000","last_became_peered":"2026-02-20T21:03:08.051990+0000","last_unstale":"2026-02-20T21:28:06.451888+0000","last_undegraded":"2026-02-20T21:28:06.451888+0000","last_fullsized":"2026-02-20T21:28:06.451888+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:28:06.451807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:28:06.451807+0000","last_clean_scrub_stamp":"2026-02-20T21:28:06.451807+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:10:56.969732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050303499999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1138,"reported_epoch":565,"state":"active+clean","last_fresh":"2026-02-20T21:28:05.416673+0000","last_change":"2026-02-20T21:28:05.416615+0000","last_active":"2026-02-20T21:28:05.416673+0000","last_peered":"2026-02-20T21:28:05.416673+0000","last_clean":"2026-02-20T21:28:05.416673+0000","last_became_active":"2026-02-20T21:03:08.052009+0000","last_became_peered":"2026-02-20T21:03:08.052009+0000","last_unstale":"2026-02-20T21:28:05.416673+0000","last_undegraded":"2026-02-20T21:28:05.416673+0000","last_fullsized":"2026-02-20T21:28:05.416673+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:28:05.416554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:28:05.416554+0000","last_clean_scrub_stamp":"2026-02-20T21:28:05.416554+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T08:49:06.353641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039422300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"565'3","reported_seq":1141,"reported_epoch":565,"state":"active+clean","last_fresh":"2026-02-20T21:28:04.236031+0000","last_change":"2026-02-20T21:28:04.235988+0000","last_active":"2026-02-20T21:28:04.236031+0000","last_peered":"2026-02-20T21:28:04.236031+0000","last_clean":"2026-02-20T21:28:04.236031+0000","last_became_active":"2026-02-20T21:03:08.052246+0000","last_became_peered":"2026-02-20T21:03:08.052246+0000","last_unstale":"2026-02-20T21:28:04.236031+0000","last_undegraded":"2026-02-20T21:28:04.236031+0000","last_fullsized":"2026-02-20T21:28:04.236031+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"565'3","last_scrub_stamp":"2026-02-20T21:28:04.235942+0000","last_deep_scrub":"565'3","last_deep_scrub_stamp":"2026-02-20T21:28:04.235942+0000","last_clean_scrub_stamp":"2026-02-20T21:28:04.235942+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:57:44.206064+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00076615400000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1137,"reported_epoch":565,"state":"active+clean","last_fresh":"2026-02-20T21:28:05.656307+0000","last_change":"2026-02-20T21:28:05.656246+0000","last_active":"2026-02-20T21:28:05.656307+0000","last_peered":"2026-02-20T21:28:05.656307+0000","last_clean":"2026-02-20T21:28:05.656307+0000","last_became_active":"2026-02-20T21:03:08.052284+0000","last_became_peered":"2026-02-20T21:03:08.052284+0000","last_unstale":"2026-02-20T21:28:05.656307+0000","last_undegraded":"2026-02-20T21:28:05.656307+0000","last_fullsized":"2026-02-20T21:28:05.656307+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:28:05.656177+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:28:05.656177+0000","last_clean_scrub_stamp":"2026-02-20T21:28:05.656177+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:37:51.448331+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00030119000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1137,"reported_epoch":565,"state":"active+clean","last_fresh":"2026-02-20T21:28:04.701014+0000","last_change":"2026-02-20T21:28:04.701014+0000","last_active":"2026-02-20T21:28:04.701014+0000","last_peered":"2026-02-20T21:28:04.701014+0000","last_clean":"2026-02-20T21:28:04.701014+0000","last_became_active":"2026-02-20T21:03:08.052106+0000","last_became_peered":"2026-02-20T21:03:08.052106+0000","last_unstale":"2026-02-20T21:28:04.701014+0000","last_undegraded":"2026-02-20T21:28:04.701014+0000","last_fullsized":"2026-02-20T21:28:04.701014+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:28:04.700959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:28:04.700959+0000","last_clean_scrub_stamp":"2026-02-20T21:28:04.700959+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:49:03.443575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000257268,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"565'2","reported_seq":1140,"reported_epoch":565,"state":"active+clean","last_fresh":"2026-02-20T21:28:04.461019+0000","last_change":"2026-02-20T21:28:04.460977+0000","last_active":"2026-02-20T21:28:04.461019+0000","last_peered":"2026-02-20T21:28:04.461019+0000","last_clean":"2026-02-20T21:28:04.461019+0000","last_became_active":"2026-02-20T21:03:08.051931+0000","last_became_peered":"2026-02-20T21:03:08.051931+0000","last_unstale":"2026-02-20T21:28:04.461019+0000","last_undegraded":"2026-02-20T21:28:04.461019+0000","last_fullsized":"2026-02-20T21:28:04.461019+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"565'2","last_scrub_stamp":"2026-02-20T21:28:04.460923+0000","last_deep_scrub":"565'2","last_deep_scrub_stamp":"2026-02-20T21:28:04.460923+0000","last_clean_scrub_stamp":"2026-02-20T21:28:04.460923+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:01:45.164104+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.000310737,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"565'74","reported_seq":1231,"reported_epoch":565,"state":"active+clean","last_fresh":"2026-02-20T21:28:07.494754+0000","last_change":"2026-02-20T21:28:07.494754+0000","last_active":"2026-02-20T21:28:07.494754+0000","last_peered":"2026-02-20T21:28:07.494754+0000","last_clean":"2026-02-20T21:28:07.494754+0000","last_became_active":"2026-02-20T21:03:05.421208+0000","last_became_peered":"2026-02-20T21:03:05.421208+0000","last_unstale":"2026-02-20T21:28:07.494754+0000","last_undegraded":"2026-02-20T21:28:07.494754+0000","last_fullsized":"2026-02-20T21:28:07.494754+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"565'74","last_scrub_stamp":"2026-02-20T21:28:07.494712+0000","last_deep_scrub":"565'74","last_deep_scrub_stamp":"2026-02-20T21:28:07.494712+0000","last_clean_scrub_stamp":"2026-02-20T21:28:07.494712+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:20:03.143059+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67134,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738675,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":139720,"kb_used_data":4536,"kb_used_omap":2073,"kb_used_meta":133094,"kb_avail":94232120,"statfs":{"total":96636764160,"available":96493690880,"internally_reserved":0,"allocated":4644864,"data_stored":2489738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2123596,"internal_metadata":136288436},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738676,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":193996,"kb_used_data":4604,"kb_used_omap":2115,"kb_used_meta":187260,"kb_avail":94177844,"statfs":{"total":96636764160,"available":96438112256,"internally_reserved":0,"allocated":4714496,"data_stored":3080125,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":2166315,"internal_metadata":191754709},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738676,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":175820,"kb_used_data":4604,"kb_used_omap":2603,"kb_used_meta":168596,"kb_avail":94196020,"statfs":{"total":96636764160,"available":96456724480,"internally_reserved":0,"allocated":4714496,"data_stored":3080125,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":2665853,"internal_metadata":172642947},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33567,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:28:24.753 DEBUG:teuthology.orchestra.run.trial199:> sudo ceph --cluster cluster2 config set global mon_health_to_clog false 2026-02-20T21:28:24.943 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-20T21:28:24.943 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-20T21:28:24.944 DEBUG:tasks.ceph.cluster2.osd.0:waiting for process to exit 2026-02-20T21:28:24.944 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:28:24.989 INFO:tasks.ceph.cluster2.osd.0:Stopped 2026-02-20T21:28:24.989 DEBUG:tasks.ceph.cluster2.osd.1:waiting for process to exit 2026-02-20T21:28:24.990 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:28:25.029 INFO:tasks.ceph.cluster2.osd.1:Stopped 2026-02-20T21:28:25.030 DEBUG:tasks.ceph.cluster2.osd.2:waiting for process to exit 2026-02-20T21:28:25.030 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:28:25.109 INFO:tasks.ceph.cluster2.osd.2:Stopped 2026-02-20T21:28:25.109 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-20T21:28:25.109 DEBUG:tasks.ceph.cluster2.mgr.x:waiting for process to exit 2026-02-20T21:28:25.109 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:28:25.140 INFO:tasks.ceph.cluster2.mgr.x:Stopped 2026-02-20T21:28:25.140 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-20T21:28:25.140 DEBUG:tasks.ceph.cluster2.mon.a:waiting for process to exit 2026-02-20T21:28:25.141 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:28:25.160 INFO:tasks.ceph.cluster2.mon.a:Stopped 2026-02-20T21:28:25.160 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-20T21:28:25.160 DEBUG:teuthology.orchestra.run.trial199:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster2.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-20T21:28:25.225 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-0 on ubuntu@trial199.front.sepia.ceph.com 2026-02-20T21:28:25.226 DEBUG:teuthology.orchestra.run.trial199:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-0 2026-02-20T21:28:25.312 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-1 on ubuntu@trial199.front.sepia.ceph.com 2026-02-20T21:28:25.312 DEBUG:teuthology.orchestra.run.trial199:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-1 2026-02-20T21:28:25.414 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-2 on ubuntu@trial199.front.sepia.ceph.com 2026-02-20T21:28:25.414 DEBUG:teuthology.orchestra.run.trial199:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-2 2026-02-20T21:28:25.449 INFO:tasks.ceph:Archiving mon data... 2026-02-20T21:28:25.450 DEBUG:teuthology.misc:Transferring archived files from trial199:/var/lib/ceph/mon/cluster2-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61415/data/cluster2.mon.a.tgz 2026-02-20T21:28:25.450 DEBUG:teuthology.orchestra.run.trial199:> mktemp 2026-02-20T21:28:25.504 INFO:teuthology.orchestra.run.trial199.stdout:/tmp/tmp.45dG0yeA0H 2026-02-20T21:28:25.505 DEBUG:teuthology.orchestra.run.trial199:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster2-a -- . > /tmp/tmp.45dG0yeA0H 2026-02-20T21:28:25.627 DEBUG:teuthology.orchestra.run.trial199:> sudo chmod 0666 /tmp/tmp.45dG0yeA0H 2026-02-20T21:28:25.708 DEBUG:teuthology.orchestra.remote:trial199:/tmp/tmp.45dG0yeA0H is 468KB 2026-02-20T21:28:25.805 DEBUG:teuthology.orchestra.run.trial199:> rm -fr /tmp/tmp.45dG0yeA0H 2026-02-20T21:28:25.822 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-20T21:28:25.823 DEBUG:teuthology.orchestra.run.trial196:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-20T21:28:25.826 DEBUG:teuthology.orchestra.run.trial199:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-20T21:28:25.886 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-20T21:28:25.898 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-20T21:28:25.898 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-20T21:28:26.080 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:28:26.080 INFO:teuthology.orchestra.run.trial196.stderr:dumped all 2026-02-20T21:28:26.089 INFO:teuthology.orchestra.run.trial196.stdout:{"pg_ready":true,"pg_map":{"version":1351,"stamp":"2026-02-20T21:28:25.263089+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":365940,"kb_used_data":14404,"kb_used_omap":4599,"kb_used_meta":346888,"kb_avail":282749580,"statfs":{"total":289910292480,"available":289535569920,"internally_reserved":0,"allocated":14749696,"data_stored":8898661,"data_compressed":67146,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":4710196,"internal_metadata":355213516},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000970"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1161,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.042085+0000","last_change":"2026-02-20T21:02:44.848549+0000","last_active":"2026-02-20T21:27:59.042085+0000","last_peered":"2026-02-20T21:27:59.042085+0000","last_clean":"2026-02-20T21:27:59.042085+0000","last_became_active":"2026-02-20T21:02:42.847423+0000","last_became_peered":"2026-02-20T21:02:42.847423+0000","last_unstale":"2026-02-20T21:27:59.042085+0000","last_undegraded":"2026-02-20T21:27:59.042085+0000","last_fullsized":"2026-02-20T21:27:59.042085+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:11:14.799466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000806229,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1161,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.044088+0000","last_change":"2026-02-20T21:02:44.848660+0000","last_active":"2026-02-20T21:27:59.044088+0000","last_peered":"2026-02-20T21:27:59.044088+0000","last_clean":"2026-02-20T21:27:59.044088+0000","last_became_active":"2026-02-20T21:02:42.847356+0000","last_became_peered":"2026-02-20T21:02:42.847356+0000","last_unstale":"2026-02-20T21:27:59.044088+0000","last_undegraded":"2026-02-20T21:27:59.044088+0000","last_fullsized":"2026-02-20T21:27:59.044088+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:12:14.023391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00098168499999999994,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1162,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.039744+0000","last_change":"2026-02-20T21:02:44.848481+0000","last_active":"2026-02-20T21:27:59.039744+0000","last_peered":"2026-02-20T21:27:59.039744+0000","last_clean":"2026-02-20T21:27:59.039744+0000","last_became_active":"2026-02-20T21:02:42.847430+0000","last_became_peered":"2026-02-20T21:02:42.847430+0000","last_unstale":"2026-02-20T21:27:59.039744+0000","last_undegraded":"2026-02-20T21:27:59.039744+0000","last_fullsized":"2026-02-20T21:27:59.039744+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:35:32.634569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081569699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1162,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.042415+0000","last_change":"2026-02-20T21:02:44.848066+0000","last_active":"2026-02-20T21:27:59.042415+0000","last_peered":"2026-02-20T21:27:59.042415+0000","last_clean":"2026-02-20T21:27:59.042415+0000","last_became_active":"2026-02-20T21:02:42.847480+0000","last_became_peered":"2026-02-20T21:02:42.847480+0000","last_unstale":"2026-02-20T21:27:59.042415+0000","last_undegraded":"2026-02-20T21:27:59.042415+0000","last_fullsized":"2026-02-20T21:27:59.042415+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:56:33.514445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045216000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":1164,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.161616+0000","last_change":"2026-02-20T21:02:44.849371+0000","last_active":"2026-02-20T21:27:59.161616+0000","last_peered":"2026-02-20T21:27:59.161616+0000","last_clean":"2026-02-20T21:27:59.161616+0000","last_became_active":"2026-02-20T21:02:42.847714+0000","last_became_peered":"2026-02-20T21:02:42.847714+0000","last_unstale":"2026-02-20T21:27:59.161616+0000","last_undegraded":"2026-02-20T21:27:59.161616+0000","last_fullsized":"2026-02-20T21:27:59.161616+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:47:39.564991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089537500000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1162,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.271340+0000","last_change":"2026-02-20T21:02:45.359962+0000","last_active":"2026-02-20T21:27:59.271340+0000","last_peered":"2026-02-20T21:27:59.271340+0000","last_clean":"2026-02-20T21:27:59.271340+0000","last_became_active":"2026-02-20T21:02:42.847711+0000","last_became_peered":"2026-02-20T21:02:42.847711+0000","last_unstale":"2026-02-20T21:27:59.271340+0000","last_undegraded":"2026-02-20T21:27:59.271340+0000","last_fullsized":"2026-02-20T21:27:59.271340+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:00:52.446182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023752199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1162,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.269978+0000","last_change":"2026-02-20T21:02:45.360049+0000","last_active":"2026-02-20T21:27:59.269978+0000","last_peered":"2026-02-20T21:27:59.269978+0000","last_clean":"2026-02-20T21:27:59.269978+0000","last_became_active":"2026-02-20T21:02:42.847731+0000","last_became_peered":"2026-02-20T21:02:42.847731+0000","last_unstale":"2026-02-20T21:27:59.269978+0000","last_undegraded":"2026-02-20T21:27:59.269978+0000","last_fullsized":"2026-02-20T21:27:59.269978+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:27:20.095721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000272386,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":1163,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.043567+0000","last_change":"2026-02-20T21:02:44.848179+0000","last_active":"2026-02-20T21:27:59.043567+0000","last_peered":"2026-02-20T21:27:59.043567+0000","last_clean":"2026-02-20T21:27:59.043567+0000","last_became_active":"2026-02-20T21:02:42.847080+0000","last_became_peered":"2026-02-20T21:02:42.847080+0000","last_unstale":"2026-02-20T21:27:59.043567+0000","last_undegraded":"2026-02-20T21:27:59.043567+0000","last_fullsized":"2026-02-20T21:27:59.043567+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:43:24.107388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057432800000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":1252,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.043675+0000","last_change":"2026-02-20T21:02:39.094532+0000","last_active":"2026-02-20T21:27:59.043675+0000","last_peered":"2026-02-20T21:27:59.043675+0000","last_clean":"2026-02-20T21:27:59.043675+0000","last_became_active":"2026-02-20T21:02:39.093868+0000","last_became_peered":"2026-02-20T21:02:39.093868+0000","last_unstale":"2026-02-20T21:27:59.043675+0000","last_undegraded":"2026-02-20T21:27:59.043675+0000","last_fullsized":"2026-02-20T21:27:59.043675+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:37.832019+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:37.832019+0000","last_clean_scrub_stamp":"2026-02-20T21:02:37.832019+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:52:04.367262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67146,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738681,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":103844,"kb_used_data":4756,"kb_used_omap":1724,"kb_used_meta":97347,"kb_avail":94267996,"statfs":{"total":96636764160,"available":96530427904,"internally_reserved":0,"allocated":4870144,"data_stored":2572629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1765539,"internal_metadata":99684189},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738679,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":103976,"kb_used_data":4824,"kb_used_omap":1094,"kb_used_meta":98041,"kb_avail":94267864,"statfs":{"total":96636764160,"available":96530292736,"internally_reserved":0,"allocated":4939776,"data_stored":3163016,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":1121027,"internal_metadata":100394237},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738679,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":158120,"kb_used_data":4824,"kb_used_omap":1780,"kb_used_meta":151499,"kb_avail":94213720,"statfs":{"total":96636764160,"available":96474849280,"internally_reserved":0,"allocated":4939776,"data_stored":3163016,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":1823630,"internal_metadata":155135090},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:28:26.091 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-20T21:28:26.270 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:28:26.271 INFO:teuthology.orchestra.run.trial196.stderr:dumped all 2026-02-20T21:28:26.279 INFO:teuthology.orchestra.run.trial196.stdout:{"pg_ready":true,"pg_map":{"version":1351,"stamp":"2026-02-20T21:28:25.263089+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":365940,"kb_used_data":14404,"kb_used_omap":4599,"kb_used_meta":346888,"kb_avail":282749580,"statfs":{"total":289910292480,"available":289535569920,"internally_reserved":0,"allocated":14749696,"data_stored":8898661,"data_compressed":67146,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":4710196,"internal_metadata":355213516},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000970"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1161,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.042085+0000","last_change":"2026-02-20T21:02:44.848549+0000","last_active":"2026-02-20T21:27:59.042085+0000","last_peered":"2026-02-20T21:27:59.042085+0000","last_clean":"2026-02-20T21:27:59.042085+0000","last_became_active":"2026-02-20T21:02:42.847423+0000","last_became_peered":"2026-02-20T21:02:42.847423+0000","last_unstale":"2026-02-20T21:27:59.042085+0000","last_undegraded":"2026-02-20T21:27:59.042085+0000","last_fullsized":"2026-02-20T21:27:59.042085+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:11:14.799466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000806229,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1161,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.044088+0000","last_change":"2026-02-20T21:02:44.848660+0000","last_active":"2026-02-20T21:27:59.044088+0000","last_peered":"2026-02-20T21:27:59.044088+0000","last_clean":"2026-02-20T21:27:59.044088+0000","last_became_active":"2026-02-20T21:02:42.847356+0000","last_became_peered":"2026-02-20T21:02:42.847356+0000","last_unstale":"2026-02-20T21:27:59.044088+0000","last_undegraded":"2026-02-20T21:27:59.044088+0000","last_fullsized":"2026-02-20T21:27:59.044088+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:12:14.023391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00098168499999999994,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1162,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.039744+0000","last_change":"2026-02-20T21:02:44.848481+0000","last_active":"2026-02-20T21:27:59.039744+0000","last_peered":"2026-02-20T21:27:59.039744+0000","last_clean":"2026-02-20T21:27:59.039744+0000","last_became_active":"2026-02-20T21:02:42.847430+0000","last_became_peered":"2026-02-20T21:02:42.847430+0000","last_unstale":"2026-02-20T21:27:59.039744+0000","last_undegraded":"2026-02-20T21:27:59.039744+0000","last_fullsized":"2026-02-20T21:27:59.039744+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:35:32.634569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081569699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1162,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.042415+0000","last_change":"2026-02-20T21:02:44.848066+0000","last_active":"2026-02-20T21:27:59.042415+0000","last_peered":"2026-02-20T21:27:59.042415+0000","last_clean":"2026-02-20T21:27:59.042415+0000","last_became_active":"2026-02-20T21:02:42.847480+0000","last_became_peered":"2026-02-20T21:02:42.847480+0000","last_unstale":"2026-02-20T21:27:59.042415+0000","last_undegraded":"2026-02-20T21:27:59.042415+0000","last_fullsized":"2026-02-20T21:27:59.042415+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:56:33.514445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045216000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":1164,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.161616+0000","last_change":"2026-02-20T21:02:44.849371+0000","last_active":"2026-02-20T21:27:59.161616+0000","last_peered":"2026-02-20T21:27:59.161616+0000","last_clean":"2026-02-20T21:27:59.161616+0000","last_became_active":"2026-02-20T21:02:42.847714+0000","last_became_peered":"2026-02-20T21:02:42.847714+0000","last_unstale":"2026-02-20T21:27:59.161616+0000","last_undegraded":"2026-02-20T21:27:59.161616+0000","last_fullsized":"2026-02-20T21:27:59.161616+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:47:39.564991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089537500000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1162,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.271340+0000","last_change":"2026-02-20T21:02:45.359962+0000","last_active":"2026-02-20T21:27:59.271340+0000","last_peered":"2026-02-20T21:27:59.271340+0000","last_clean":"2026-02-20T21:27:59.271340+0000","last_became_active":"2026-02-20T21:02:42.847711+0000","last_became_peered":"2026-02-20T21:02:42.847711+0000","last_unstale":"2026-02-20T21:27:59.271340+0000","last_undegraded":"2026-02-20T21:27:59.271340+0000","last_fullsized":"2026-02-20T21:27:59.271340+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:00:52.446182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023752199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1162,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.269978+0000","last_change":"2026-02-20T21:02:45.360049+0000","last_active":"2026-02-20T21:27:59.269978+0000","last_peered":"2026-02-20T21:27:59.269978+0000","last_clean":"2026-02-20T21:27:59.269978+0000","last_became_active":"2026-02-20T21:02:42.847731+0000","last_became_peered":"2026-02-20T21:02:42.847731+0000","last_unstale":"2026-02-20T21:27:59.269978+0000","last_undegraded":"2026-02-20T21:27:59.269978+0000","last_fullsized":"2026-02-20T21:27:59.269978+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:27:20.095721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000272386,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":1163,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.043567+0000","last_change":"2026-02-20T21:02:44.848179+0000","last_active":"2026-02-20T21:27:59.043567+0000","last_peered":"2026-02-20T21:27:59.043567+0000","last_clean":"2026-02-20T21:27:59.043567+0000","last_became_active":"2026-02-20T21:02:42.847080+0000","last_became_peered":"2026-02-20T21:02:42.847080+0000","last_unstale":"2026-02-20T21:27:59.043567+0000","last_undegraded":"2026-02-20T21:27:59.043567+0000","last_fullsized":"2026-02-20T21:27:59.043567+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:43:24.107388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057432800000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":1252,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.043675+0000","last_change":"2026-02-20T21:02:39.094532+0000","last_active":"2026-02-20T21:27:59.043675+0000","last_peered":"2026-02-20T21:27:59.043675+0000","last_clean":"2026-02-20T21:27:59.043675+0000","last_became_active":"2026-02-20T21:02:39.093868+0000","last_became_peered":"2026-02-20T21:02:39.093868+0000","last_unstale":"2026-02-20T21:27:59.043675+0000","last_undegraded":"2026-02-20T21:27:59.043675+0000","last_fullsized":"2026-02-20T21:27:59.043675+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:37.832019+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:37.832019+0000","last_clean_scrub_stamp":"2026-02-20T21:02:37.832019+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:52:04.367262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67146,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738681,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":103844,"kb_used_data":4756,"kb_used_omap":1724,"kb_used_meta":97347,"kb_avail":94267996,"statfs":{"total":96636764160,"available":96530427904,"internally_reserved":0,"allocated":4870144,"data_stored":2572629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1765539,"internal_metadata":99684189},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738679,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":103976,"kb_used_data":4824,"kb_used_omap":1094,"kb_used_meta":98041,"kb_avail":94267864,"statfs":{"total":96636764160,"available":96530292736,"internally_reserved":0,"allocated":4939776,"data_stored":3163016,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":1121027,"internal_metadata":100394237},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738679,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":158120,"kb_used_data":4824,"kb_used_omap":1780,"kb_used_meta":151499,"kb_avail":94213720,"statfs":{"total":96636764160,"available":96474849280,"internally_reserved":0,"allocated":4939776,"data_stored":3163016,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":1823630,"internal_metadata":155135090},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:28:26.280 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-20T21:28:26.281 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-20T21:28:26.461 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:28:26.461 INFO:teuthology.orchestra.run.trial196.stderr:dumped all 2026-02-20T21:28:26.470 INFO:teuthology.orchestra.run.trial196.stdout:{"pg_ready":true,"pg_map":{"version":1351,"stamp":"2026-02-20T21:28:25.263089+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":365940,"kb_used_data":14404,"kb_used_omap":4599,"kb_used_meta":346888,"kb_avail":282749580,"statfs":{"total":289910292480,"available":289535569920,"internally_reserved":0,"allocated":14749696,"data_stored":8898661,"data_compressed":67146,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":4710196,"internal_metadata":355213516},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000970"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1161,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.042085+0000","last_change":"2026-02-20T21:02:44.848549+0000","last_active":"2026-02-20T21:27:59.042085+0000","last_peered":"2026-02-20T21:27:59.042085+0000","last_clean":"2026-02-20T21:27:59.042085+0000","last_became_active":"2026-02-20T21:02:42.847423+0000","last_became_peered":"2026-02-20T21:02:42.847423+0000","last_unstale":"2026-02-20T21:27:59.042085+0000","last_undegraded":"2026-02-20T21:27:59.042085+0000","last_fullsized":"2026-02-20T21:27:59.042085+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:11:14.799466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000806229,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1161,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.044088+0000","last_change":"2026-02-20T21:02:44.848660+0000","last_active":"2026-02-20T21:27:59.044088+0000","last_peered":"2026-02-20T21:27:59.044088+0000","last_clean":"2026-02-20T21:27:59.044088+0000","last_became_active":"2026-02-20T21:02:42.847356+0000","last_became_peered":"2026-02-20T21:02:42.847356+0000","last_unstale":"2026-02-20T21:27:59.044088+0000","last_undegraded":"2026-02-20T21:27:59.044088+0000","last_fullsized":"2026-02-20T21:27:59.044088+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:12:14.023391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00098168499999999994,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1162,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.039744+0000","last_change":"2026-02-20T21:02:44.848481+0000","last_active":"2026-02-20T21:27:59.039744+0000","last_peered":"2026-02-20T21:27:59.039744+0000","last_clean":"2026-02-20T21:27:59.039744+0000","last_became_active":"2026-02-20T21:02:42.847430+0000","last_became_peered":"2026-02-20T21:02:42.847430+0000","last_unstale":"2026-02-20T21:27:59.039744+0000","last_undegraded":"2026-02-20T21:27:59.039744+0000","last_fullsized":"2026-02-20T21:27:59.039744+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:35:32.634569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081569699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1162,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.042415+0000","last_change":"2026-02-20T21:02:44.848066+0000","last_active":"2026-02-20T21:27:59.042415+0000","last_peered":"2026-02-20T21:27:59.042415+0000","last_clean":"2026-02-20T21:27:59.042415+0000","last_became_active":"2026-02-20T21:02:42.847480+0000","last_became_peered":"2026-02-20T21:02:42.847480+0000","last_unstale":"2026-02-20T21:27:59.042415+0000","last_undegraded":"2026-02-20T21:27:59.042415+0000","last_fullsized":"2026-02-20T21:27:59.042415+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:56:33.514445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045216000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":1164,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.161616+0000","last_change":"2026-02-20T21:02:44.849371+0000","last_active":"2026-02-20T21:27:59.161616+0000","last_peered":"2026-02-20T21:27:59.161616+0000","last_clean":"2026-02-20T21:27:59.161616+0000","last_became_active":"2026-02-20T21:02:42.847714+0000","last_became_peered":"2026-02-20T21:02:42.847714+0000","last_unstale":"2026-02-20T21:27:59.161616+0000","last_undegraded":"2026-02-20T21:27:59.161616+0000","last_fullsized":"2026-02-20T21:27:59.161616+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:47:39.564991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089537500000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1162,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.271340+0000","last_change":"2026-02-20T21:02:45.359962+0000","last_active":"2026-02-20T21:27:59.271340+0000","last_peered":"2026-02-20T21:27:59.271340+0000","last_clean":"2026-02-20T21:27:59.271340+0000","last_became_active":"2026-02-20T21:02:42.847711+0000","last_became_peered":"2026-02-20T21:02:42.847711+0000","last_unstale":"2026-02-20T21:27:59.271340+0000","last_undegraded":"2026-02-20T21:27:59.271340+0000","last_fullsized":"2026-02-20T21:27:59.271340+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:00:52.446182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023752199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1162,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.269978+0000","last_change":"2026-02-20T21:02:45.360049+0000","last_active":"2026-02-20T21:27:59.269978+0000","last_peered":"2026-02-20T21:27:59.269978+0000","last_clean":"2026-02-20T21:27:59.269978+0000","last_became_active":"2026-02-20T21:02:42.847731+0000","last_became_peered":"2026-02-20T21:02:42.847731+0000","last_unstale":"2026-02-20T21:27:59.269978+0000","last_undegraded":"2026-02-20T21:27:59.269978+0000","last_fullsized":"2026-02-20T21:27:59.269978+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:27:20.095721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000272386,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":1163,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.043567+0000","last_change":"2026-02-20T21:02:44.848179+0000","last_active":"2026-02-20T21:27:59.043567+0000","last_peered":"2026-02-20T21:27:59.043567+0000","last_clean":"2026-02-20T21:27:59.043567+0000","last_became_active":"2026-02-20T21:02:42.847080+0000","last_became_peered":"2026-02-20T21:02:42.847080+0000","last_unstale":"2026-02-20T21:27:59.043567+0000","last_undegraded":"2026-02-20T21:27:59.043567+0000","last_fullsized":"2026-02-20T21:27:59.043567+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:43:24.107388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057432800000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":1252,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.043675+0000","last_change":"2026-02-20T21:02:39.094532+0000","last_active":"2026-02-20T21:27:59.043675+0000","last_peered":"2026-02-20T21:27:59.043675+0000","last_clean":"2026-02-20T21:27:59.043675+0000","last_became_active":"2026-02-20T21:02:39.093868+0000","last_became_peered":"2026-02-20T21:02:39.093868+0000","last_unstale":"2026-02-20T21:27:59.043675+0000","last_undegraded":"2026-02-20T21:27:59.043675+0000","last_fullsized":"2026-02-20T21:27:59.043675+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:37.832019+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:37.832019+0000","last_clean_scrub_stamp":"2026-02-20T21:02:37.832019+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:52:04.367262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67146,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738681,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":103844,"kb_used_data":4756,"kb_used_omap":1724,"kb_used_meta":97347,"kb_avail":94267996,"statfs":{"total":96636764160,"available":96530427904,"internally_reserved":0,"allocated":4870144,"data_stored":2572629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1765539,"internal_metadata":99684189},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738679,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":103976,"kb_used_data":4824,"kb_used_omap":1094,"kb_used_meta":98041,"kb_avail":94267864,"statfs":{"total":96636764160,"available":96530292736,"internally_reserved":0,"allocated":4939776,"data_stored":3163016,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":1121027,"internal_metadata":100394237},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738679,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":158120,"kb_used_data":4824,"kb_used_omap":1780,"kb_used_meta":151499,"kb_avail":94213720,"statfs":{"total":96636764160,"available":96474849280,"internally_reserved":0,"allocated":4939776,"data_stored":3163016,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":1823630,"internal_metadata":155135090},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:28:26.472 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-20T21:28:26.654 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:28:26.654 INFO:teuthology.orchestra.run.trial196.stdout:{"epoch":580,"fsid":"d723a433-cea3-4643-ada0-27ddcd59cc6f","created":"2026-02-20T21:02:32.822606+0000","modified":"2026-02-20T21:27:59.037358+0000","last_up_change":"2026-02-20T21:02:36.830199+0000","last_in_change":"2026-02-20T21:02:33.507752+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:02:37.069741+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-20T21:02:40.909401+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"3daca0a3-5551-443d-aa67-7bfefab90bd6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6808","nonce":1534278585},{"type":"v1","addr":"10.20.193.196:6810","nonce":1534278585}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6812","nonce":1534278585},{"type":"v1","addr":"10.20.193.196:6815","nonce":1534278585}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6821","nonce":1534278585},{"type":"v1","addr":"10.20.193.196:6823","nonce":1534278585}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6817","nonce":1534278585},{"type":"v1","addr":"10.20.193.196:6819","nonce":1534278585}]},"public_addr":"10.20.193.196:6810/1534278585","cluster_addr":"10.20.193.196:6815/1534278585","heartbeat_back_addr":"10.20.193.196:6823/1534278585","heartbeat_front_addr":"10.20.193.196:6819/1534278585","state":["exists","up"]},{"osd":1,"uuid":"68aa9c95-119c-4129-83d1-d79efba07215","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6809","nonce":3174935431},{"type":"v1","addr":"10.20.193.196:6811","nonce":3174935431}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6813","nonce":3174935431},{"type":"v1","addr":"10.20.193.196:6814","nonce":3174935431}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6820","nonce":3174935431},{"type":"v1","addr":"10.20.193.196:6822","nonce":3174935431}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6816","nonce":3174935431},{"type":"v1","addr":"10.20.193.196:6818","nonce":3174935431}]},"public_addr":"10.20.193.196:6811/3174935431","cluster_addr":"10.20.193.196:6814/3174935431","heartbeat_back_addr":"10.20.193.196:6822/3174935431","heartbeat_front_addr":"10.20.193.196:6818/3174935431","state":["exists","up"]},{"osd":2,"uuid":"c09e5039-21ad-44d7-a5c8-b008a89ac5b4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6800","nonce":2134765115},{"type":"v1","addr":"10.20.193.196:6801","nonce":2134765115}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6802","nonce":2134765115},{"type":"v1","addr":"10.20.193.196:6803","nonce":2134765115}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6806","nonce":2134765115},{"type":"v1","addr":"10.20.193.196:6807","nonce":2134765115}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.196:6804","nonce":2134765115},{"type":"v1","addr":"10.20.193.196:6805","nonce":2134765115}]},"public_addr":"10.20.193.196:6801/2134765115","cluster_addr":"10.20.193.196:6803/2134765115","heartbeat_back_addr":"10.20.193.196:6807/2134765115","heartbeat_front_addr":"10.20.193.196:6805/2134765115","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:02:35.468565+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:02:35.469621+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:02:35.434924+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"10.20.193.199:0/1680218099":"2026-02-20T22:27:37.006445+0000","10.20.193.199:0/1518997133":"2026-02-20T22:27:37.007001+0000","10.20.193.199:0/134008187":"2026-02-20T22:27:37.006634+0000","10.20.193.199:0/3941798472":"2026-02-20T22:25:03.788267+0000","10.20.193.199:0/1275424490":"2026-02-20T22:25:03.808015+0000","10.20.193.199:0/954511169":"2026-02-20T22:25:33.989376+0000","10.20.193.199:0/2717410102":"2026-02-20T22:27:37.007202+0000","10.20.193.199:0/1781790308":"2026-02-20T22:27:06.771346+0000","10.20.193.199:0/1056347611":"2026-02-20T22:25:33.989317+0000","10.20.193.199:0/738883253":"2026-02-20T22:25:33.988546+0000","10.20.193.199:0/368540168":"2026-02-20T22:27:00.160190+0000","10.20.193.199:0/1417630852":"2026-02-20T22:25:33.988607+0000","10.20.193.199:0/1840933383":"2026-02-20T22:27:37.007132+0000","10.20.193.199:0/408132056":"2026-02-20T22:25:33.988465+0000","10.20.193.199:0/3030755295":"2026-02-20T22:25:33.989247+0000","10.20.193.199:0/2478317165":"2026-02-20T22:27:37.006713+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":217,"length":1},{"begin":220,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:28:27.664 INFO:tasks.ceph:Scrubbing cluster1.osd.0 2026-02-20T21:28:27.664 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-20T21:28:27.748 INFO:teuthology.orchestra.run.trial196.stdout:{ 2026-02-20T21:28:27.748 INFO:teuthology.orchestra.run.trial196.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-20T21:28:27.748 INFO:teuthology.orchestra.run.trial196.stdout:} 2026-02-20T21:28:27.754 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 0 2026-02-20T21:28:27.936 INFO:teuthology.orchestra.run.trial196.stderr:instructed osd(s) 0 to deep-scrub 2026-02-20T21:28:27.945 INFO:tasks.ceph:Scrubbing cluster1.osd.1 2026-02-20T21:28:27.945 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-20T21:28:28.026 INFO:teuthology.orchestra.run.trial196.stdout:{ 2026-02-20T21:28:28.027 INFO:teuthology.orchestra.run.trial196.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-20T21:28:28.027 INFO:teuthology.orchestra.run.trial196.stdout:} 2026-02-20T21:28:28.032 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 1 2026-02-20T21:28:28.214 INFO:teuthology.orchestra.run.trial196.stderr:instructed osd(s) 1 to deep-scrub 2026-02-20T21:28:28.223 INFO:tasks.ceph:Scrubbing cluster1.osd.2 2026-02-20T21:28:28.224 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-20T21:28:28.305 INFO:teuthology.orchestra.run.trial196.stdout:{ 2026-02-20T21:28:28.305 INFO:teuthology.orchestra.run.trial196.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-20T21:28:28.306 INFO:teuthology.orchestra.run.trial196.stdout:} 2026-02-20T21:28:28.312 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 2 2026-02-20T21:28:28.493 INFO:teuthology.orchestra.run.trial196.stderr:instructed osd(s) 2 to deep-scrub 2026-02-20T21:28:28.502 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-20T21:28:28.680 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:28:28.681 INFO:teuthology.orchestra.run.trial196.stderr:dumped all 2026-02-20T21:28:28.689 INFO:teuthology.orchestra.run.trial196.stdout:{"pg_ready":true,"pg_map":{"version":1352,"stamp":"2026-02-20T21:28:27.263321+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":365940,"kb_used_data":14404,"kb_used_omap":4599,"kb_used_meta":346888,"kb_avail":282749580,"statfs":{"total":289910292480,"available":289535569920,"internally_reserved":0,"allocated":14749696,"data_stored":8898661,"data_compressed":67146,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":4710196,"internal_metadata":355213516},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001028"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1161,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.042085+0000","last_change":"2026-02-20T21:02:44.848549+0000","last_active":"2026-02-20T21:27:59.042085+0000","last_peered":"2026-02-20T21:27:59.042085+0000","last_clean":"2026-02-20T21:27:59.042085+0000","last_became_active":"2026-02-20T21:02:42.847423+0000","last_became_peered":"2026-02-20T21:02:42.847423+0000","last_unstale":"2026-02-20T21:27:59.042085+0000","last_undegraded":"2026-02-20T21:27:59.042085+0000","last_fullsized":"2026-02-20T21:27:59.042085+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:11:14.799466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000806229,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1161,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.044088+0000","last_change":"2026-02-20T21:02:44.848660+0000","last_active":"2026-02-20T21:27:59.044088+0000","last_peered":"2026-02-20T21:27:59.044088+0000","last_clean":"2026-02-20T21:27:59.044088+0000","last_became_active":"2026-02-20T21:02:42.847356+0000","last_became_peered":"2026-02-20T21:02:42.847356+0000","last_unstale":"2026-02-20T21:27:59.044088+0000","last_undegraded":"2026-02-20T21:27:59.044088+0000","last_fullsized":"2026-02-20T21:27:59.044088+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:12:14.023391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00098168499999999994,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1162,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.039744+0000","last_change":"2026-02-20T21:02:44.848481+0000","last_active":"2026-02-20T21:27:59.039744+0000","last_peered":"2026-02-20T21:27:59.039744+0000","last_clean":"2026-02-20T21:27:59.039744+0000","last_became_active":"2026-02-20T21:02:42.847430+0000","last_became_peered":"2026-02-20T21:02:42.847430+0000","last_unstale":"2026-02-20T21:27:59.039744+0000","last_undegraded":"2026-02-20T21:27:59.039744+0000","last_fullsized":"2026-02-20T21:27:59.039744+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:35:32.634569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081569699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1162,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.042415+0000","last_change":"2026-02-20T21:02:44.848066+0000","last_active":"2026-02-20T21:27:59.042415+0000","last_peered":"2026-02-20T21:27:59.042415+0000","last_clean":"2026-02-20T21:27:59.042415+0000","last_became_active":"2026-02-20T21:02:42.847480+0000","last_became_peered":"2026-02-20T21:02:42.847480+0000","last_unstale":"2026-02-20T21:27:59.042415+0000","last_undegraded":"2026-02-20T21:27:59.042415+0000","last_fullsized":"2026-02-20T21:27:59.042415+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:56:33.514445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045216000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":1164,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.161616+0000","last_change":"2026-02-20T21:02:44.849371+0000","last_active":"2026-02-20T21:27:59.161616+0000","last_peered":"2026-02-20T21:27:59.161616+0000","last_clean":"2026-02-20T21:27:59.161616+0000","last_became_active":"2026-02-20T21:02:42.847714+0000","last_became_peered":"2026-02-20T21:02:42.847714+0000","last_unstale":"2026-02-20T21:27:59.161616+0000","last_undegraded":"2026-02-20T21:27:59.161616+0000","last_fullsized":"2026-02-20T21:27:59.161616+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:47:39.564991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089537500000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1162,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.271340+0000","last_change":"2026-02-20T21:02:45.359962+0000","last_active":"2026-02-20T21:27:59.271340+0000","last_peered":"2026-02-20T21:27:59.271340+0000","last_clean":"2026-02-20T21:27:59.271340+0000","last_became_active":"2026-02-20T21:02:42.847711+0000","last_became_peered":"2026-02-20T21:02:42.847711+0000","last_unstale":"2026-02-20T21:27:59.271340+0000","last_undegraded":"2026-02-20T21:27:59.271340+0000","last_fullsized":"2026-02-20T21:27:59.271340+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:00:52.446182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023752199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1162,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.269978+0000","last_change":"2026-02-20T21:02:45.360049+0000","last_active":"2026-02-20T21:27:59.269978+0000","last_peered":"2026-02-20T21:27:59.269978+0000","last_clean":"2026-02-20T21:27:59.269978+0000","last_became_active":"2026-02-20T21:02:42.847731+0000","last_became_peered":"2026-02-20T21:02:42.847731+0000","last_unstale":"2026-02-20T21:27:59.269978+0000","last_undegraded":"2026-02-20T21:27:59.269978+0000","last_fullsized":"2026-02-20T21:27:59.269978+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:27:20.095721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000272386,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":1163,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.043567+0000","last_change":"2026-02-20T21:02:44.848179+0000","last_active":"2026-02-20T21:27:59.043567+0000","last_peered":"2026-02-20T21:27:59.043567+0000","last_clean":"2026-02-20T21:27:59.043567+0000","last_became_active":"2026-02-20T21:02:42.847080+0000","last_became_peered":"2026-02-20T21:02:42.847080+0000","last_unstale":"2026-02-20T21:27:59.043567+0000","last_undegraded":"2026-02-20T21:27:59.043567+0000","last_fullsized":"2026-02-20T21:27:59.043567+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:41.838625+0000","last_clean_scrub_stamp":"2026-02-20T21:02:41.838625+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:43:24.107388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057432800000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":1252,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:27:59.043675+0000","last_change":"2026-02-20T21:02:39.094532+0000","last_active":"2026-02-20T21:27:59.043675+0000","last_peered":"2026-02-20T21:27:59.043675+0000","last_clean":"2026-02-20T21:27:59.043675+0000","last_became_active":"2026-02-20T21:02:39.093868+0000","last_became_peered":"2026-02-20T21:02:39.093868+0000","last_unstale":"2026-02-20T21:27:59.043675+0000","last_undegraded":"2026-02-20T21:27:59.043675+0000","last_fullsized":"2026-02-20T21:27:59.043675+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:02:37.832019+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:02:37.832019+0000","last_clean_scrub_stamp":"2026-02-20T21:02:37.832019+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:52:04.367262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67146,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738681,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":103844,"kb_used_data":4756,"kb_used_omap":1724,"kb_used_meta":97347,"kb_avail":94267996,"statfs":{"total":96636764160,"available":96530427904,"internally_reserved":0,"allocated":4870144,"data_stored":2572629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1765539,"internal_metadata":99684189},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738679,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":103976,"kb_used_data":4824,"kb_used_omap":1094,"kb_used_meta":98041,"kb_avail":94267864,"statfs":{"total":96636764160,"available":96530292736,"internally_reserved":0,"allocated":4939776,"data_stored":3163016,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":1121027,"internal_metadata":100394237},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738680,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":158120,"kb_used_data":4824,"kb_used_omap":1780,"kb_used_meta":151499,"kb_avail":94213720,"statfs":{"total":96636764160,"available":96474849280,"internally_reserved":0,"allocated":4939776,"data_stored":3163016,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":1823630,"internal_metadata":155135090},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:28:28.691 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-20T21:02:41.838625+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=2, tm_sec=41, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=28, tm_sec=26, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:28:28.691 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-20T21:02:41.838625+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=2, tm_sec=41, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=28, tm_sec=26, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:28:28.691 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-20T21:02:41.838625+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=2, tm_sec=41, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=28, tm_sec=26, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:28:28.691 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-20T21:02:41.838625+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=2, tm_sec=41, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=28, tm_sec=26, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:28:28.691 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-20T21:02:41.838625+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=2, tm_sec=41, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=28, tm_sec=26, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:28:28.691 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-20T21:02:41.838625+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=2, tm_sec=41, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=28, tm_sec=26, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:28:28.692 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-20T21:02:41.838625+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=2, tm_sec=41, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=28, tm_sec=26, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:28:28.692 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-20T21:02:41.838625+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=2, tm_sec=41, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=28, tm_sec=26, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:28:28.692 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-20T21:02:37.832019+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=2, tm_sec=37, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=28, tm_sec=26, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:28:28.692 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-20T21:28:48.692 DEBUG:teuthology.orchestra.run.trial196:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-20T21:28:48.877 INFO:teuthology.orchestra.run.trial196.stdout: 2026-02-20T21:28:48.877 INFO:teuthology.orchestra.run.trial196.stderr:dumped all 2026-02-20T21:28:48.885 INFO:teuthology.orchestra.run.trial196.stdout:{"pg_ready":true,"pg_map":{"version":1362,"stamp":"2026-02-20T21:28:47.265048+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":365940,"kb_used_data":14404,"kb_used_omap":4599,"kb_used_meta":346888,"kb_avail":282749580,"statfs":{"total":289910292480,"available":289535569920,"internally_reserved":0,"allocated":14749696,"data_stored":8898661,"data_compressed":67146,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":4710196,"internal_metadata":355213516},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000842"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1170,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:28:32.713815+0000","last_change":"2026-02-20T21:28:32.713782+0000","last_active":"2026-02-20T21:28:32.713815+0000","last_peered":"2026-02-20T21:28:32.713815+0000","last_clean":"2026-02-20T21:28:32.713815+0000","last_became_active":"2026-02-20T21:02:42.847423+0000","last_became_peered":"2026-02-20T21:02:42.847423+0000","last_unstale":"2026-02-20T21:28:32.713815+0000","last_undegraded":"2026-02-20T21:28:32.713815+0000","last_fullsized":"2026-02-20T21:28:32.713815+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:28:32.713731+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:28:32.713731+0000","last_clean_scrub_stamp":"2026-02-20T21:28:32.713731+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T22:19:02.609167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000806229,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1170,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:28:33.679598+0000","last_change":"2026-02-20T21:28:33.679566+0000","last_active":"2026-02-20T21:28:33.679598+0000","last_peered":"2026-02-20T21:28:33.679598+0000","last_clean":"2026-02-20T21:28:33.679598+0000","last_became_active":"2026-02-20T21:02:42.847356+0000","last_became_peered":"2026-02-20T21:02:42.847356+0000","last_unstale":"2026-02-20T21:28:33.679598+0000","last_undegraded":"2026-02-20T21:28:33.679598+0000","last_fullsized":"2026-02-20T21:28:33.679598+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:28:33.679514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:28:33.679514+0000","last_clean_scrub_stamp":"2026-02-20T21:28:33.679514+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:10:20.395971+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00098168499999999994,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1170,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:28:31.670195+0000","last_change":"2026-02-20T21:28:31.670195+0000","last_active":"2026-02-20T21:28:31.670195+0000","last_peered":"2026-02-20T21:28:31.670195+0000","last_clean":"2026-02-20T21:28:31.670195+0000","last_became_active":"2026-02-20T21:02:42.847430+0000","last_became_peered":"2026-02-20T21:02:42.847430+0000","last_unstale":"2026-02-20T21:28:31.670195+0000","last_undegraded":"2026-02-20T21:28:31.670195+0000","last_fullsized":"2026-02-20T21:28:31.670195+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:28:31.670140+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:28:31.670140+0000","last_clean_scrub_stamp":"2026-02-20T21:28:31.670140+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:18:20.560486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081569699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1170,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:28:29.731983+0000","last_change":"2026-02-20T21:28:29.731983+0000","last_active":"2026-02-20T21:28:29.731983+0000","last_peered":"2026-02-20T21:28:29.731983+0000","last_clean":"2026-02-20T21:28:29.731983+0000","last_became_active":"2026-02-20T21:02:42.847480+0000","last_became_peered":"2026-02-20T21:02:42.847480+0000","last_unstale":"2026-02-20T21:28:29.731983+0000","last_undegraded":"2026-02-20T21:28:29.731983+0000","last_fullsized":"2026-02-20T21:28:29.731983+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:28:29.731937+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:28:29.731937+0000","last_clean_scrub_stamp":"2026-02-20T21:28:29.731937+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T09:01:32.433526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045216000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"580'3","reported_seq":1173,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:28:28.019543+0000","last_change":"2026-02-20T21:28:28.019543+0000","last_active":"2026-02-20T21:28:28.019543+0000","last_peered":"2026-02-20T21:28:28.019543+0000","last_clean":"2026-02-20T21:28:28.019543+0000","last_became_active":"2026-02-20T21:02:42.847714+0000","last_became_peered":"2026-02-20T21:02:42.847714+0000","last_unstale":"2026-02-20T21:28:28.019543+0000","last_undegraded":"2026-02-20T21:28:28.019543+0000","last_fullsized":"2026-02-20T21:28:28.019543+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"580'3","last_scrub_stamp":"2026-02-20T21:28:28.019501+0000","last_deep_scrub":"580'3","last_deep_scrub_stamp":"2026-02-20T21:28:28.019501+0000","last_clean_scrub_stamp":"2026-02-20T21:28:28.019501+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T00:14:49.933398+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0.00089537500000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1170,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:28:29.621193+0000","last_change":"2026-02-20T21:28:29.621193+0000","last_active":"2026-02-20T21:28:29.621193+0000","last_peered":"2026-02-20T21:28:29.621193+0000","last_clean":"2026-02-20T21:28:29.621193+0000","last_became_active":"2026-02-20T21:02:42.847711+0000","last_became_peered":"2026-02-20T21:02:42.847711+0000","last_unstale":"2026-02-20T21:28:29.621193+0000","last_undegraded":"2026-02-20T21:28:29.621193+0000","last_fullsized":"2026-02-20T21:28:29.621193+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:28:29.621142+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:28:29.621142+0000","last_clean_scrub_stamp":"2026-02-20T21:28:29.621142+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:04:40.942526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023752199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1170,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:28:28.592123+0000","last_change":"2026-02-20T21:28:28.592123+0000","last_active":"2026-02-20T21:28:28.592123+0000","last_peered":"2026-02-20T21:28:28.592123+0000","last_clean":"2026-02-20T21:28:28.592123+0000","last_became_active":"2026-02-20T21:02:42.847731+0000","last_became_peered":"2026-02-20T21:02:42.847731+0000","last_unstale":"2026-02-20T21:28:28.592123+0000","last_undegraded":"2026-02-20T21:28:28.592123+0000","last_fullsized":"2026-02-20T21:28:28.592123+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:28:28.592061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:28:28.592061+0000","last_clean_scrub_stamp":"2026-02-20T21:28:28.592061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:57:56.268466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000272386,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"580'2","reported_seq":1172,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:28:30.684600+0000","last_change":"2026-02-20T21:28:30.684600+0000","last_active":"2026-02-20T21:28:30.684600+0000","last_peered":"2026-02-20T21:28:30.684600+0000","last_clean":"2026-02-20T21:28:30.684600+0000","last_became_active":"2026-02-20T21:02:42.847080+0000","last_became_peered":"2026-02-20T21:02:42.847080+0000","last_unstale":"2026-02-20T21:28:30.684600+0000","last_undegraded":"2026-02-20T21:28:30.684600+0000","last_fullsized":"2026-02-20T21:28:30.684600+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"580'2","last_scrub_stamp":"2026-02-20T21:28:30.684550+0000","last_deep_scrub":"580'2","last_deep_scrub_stamp":"2026-02-20T21:28:30.684550+0000","last_clean_scrub_stamp":"2026-02-20T21:28:30.684550+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:29:27.029718+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00057432800000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"580'74","reported_seq":1262,"reported_epoch":580,"state":"active+clean","last_fresh":"2026-02-20T21:28:28.763603+0000","last_change":"2026-02-20T21:28:28.763603+0000","last_active":"2026-02-20T21:28:28.763603+0000","last_peered":"2026-02-20T21:28:28.763603+0000","last_clean":"2026-02-20T21:28:28.763603+0000","last_became_active":"2026-02-20T21:02:39.093868+0000","last_became_peered":"2026-02-20T21:02:39.093868+0000","last_unstale":"2026-02-20T21:28:28.763603+0000","last_undegraded":"2026-02-20T21:28:28.763603+0000","last_fullsized":"2026-02-20T21:28:28.763603+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"580'74","last_scrub_stamp":"2026-02-20T21:28:28.763575+0000","last_deep_scrub":"580'74","last_deep_scrub_stamp":"2026-02-20T21:28:28.763575+0000","last_clean_scrub_stamp":"2026-02-20T21:28:28.763575+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:03:40.261491+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67146,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738685,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":103844,"kb_used_data":4756,"kb_used_omap":1724,"kb_used_meta":97347,"kb_avail":94267996,"statfs":{"total":96636764160,"available":96530427904,"internally_reserved":0,"allocated":4870144,"data_stored":2572629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1765539,"internal_metadata":99684189},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738683,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":103976,"kb_used_data":4824,"kb_used_omap":1094,"kb_used_meta":98041,"kb_avail":94267864,"statfs":{"total":96636764160,"available":96530292736,"internally_reserved":0,"allocated":4939776,"data_stored":3163016,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":1121027,"internal_metadata":100394237},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738684,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":158120,"kb_used_data":4824,"kb_used_omap":1780,"kb_used_meta":151499,"kb_avail":94213720,"statfs":{"total":96636764160,"available":96474849280,"internally_reserved":0,"allocated":4939776,"data_stored":3163016,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":1823630,"internal_metadata":155135090},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33573,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:28:48.888 DEBUG:teuthology.orchestra.run.trial196:> sudo ceph --cluster cluster1 config set global mon_health_to_clog false 2026-02-20T21:28:49.080 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-20T21:28:49.081 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-20T21:28:49.081 DEBUG:tasks.ceph.cluster1.osd.0:waiting for process to exit 2026-02-20T21:28:49.081 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:28:49.120 INFO:tasks.ceph.cluster1.osd.0:Stopped 2026-02-20T21:28:49.120 DEBUG:tasks.ceph.cluster1.osd.1:waiting for process to exit 2026-02-20T21:28:49.121 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:28:49.196 INFO:tasks.ceph.cluster1.osd.1:Stopped 2026-02-20T21:28:49.197 DEBUG:tasks.ceph.cluster1.osd.2:waiting for process to exit 2026-02-20T21:28:49.197 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:28:49.283 INFO:tasks.ceph.cluster1.osd.2:Stopped 2026-02-20T21:28:49.284 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-20T21:28:49.284 DEBUG:tasks.ceph.cluster1.mgr.x:waiting for process to exit 2026-02-20T21:28:49.284 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:28:49.313 INFO:tasks.ceph.cluster1.mgr.x:Stopped 2026-02-20T21:28:49.313 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-20T21:28:49.313 DEBUG:tasks.ceph.cluster1.mon.a:waiting for process to exit 2026-02-20T21:28:49.313 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:28:49.364 INFO:tasks.ceph.cluster1.mon.a:Stopped 2026-02-20T21:28:49.364 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-20T21:28:49.364 DEBUG:teuthology.orchestra.run.trial196:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster1.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-20T21:28:49.430 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-0 on ubuntu@trial196.front.sepia.ceph.com 2026-02-20T21:28:49.430 DEBUG:teuthology.orchestra.run.trial196:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-0 2026-02-20T21:28:49.504 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-1 on ubuntu@trial196.front.sepia.ceph.com 2026-02-20T21:28:49.504 DEBUG:teuthology.orchestra.run.trial196:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-1 2026-02-20T21:28:49.576 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-2 on ubuntu@trial196.front.sepia.ceph.com 2026-02-20T21:28:49.576 DEBUG:teuthology.orchestra.run.trial196:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-2 2026-02-20T21:28:49.649 INFO:tasks.ceph:Archiving mon data... 2026-02-20T21:28:49.649 DEBUG:teuthology.misc:Transferring archived files from trial196:/var/lib/ceph/mon/cluster1-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61415/data/cluster1.mon.a.tgz 2026-02-20T21:28:49.650 DEBUG:teuthology.orchestra.run.trial196:> mktemp 2026-02-20T21:28:49.705 INFO:teuthology.orchestra.run.trial196.stdout:/tmp/tmp.39xRRei3Pr 2026-02-20T21:28:49.706 DEBUG:teuthology.orchestra.run.trial196:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster1-a -- . > /tmp/tmp.39xRRei3Pr 2026-02-20T21:28:49.838 DEBUG:teuthology.orchestra.run.trial196:> sudo chmod 0666 /tmp/tmp.39xRRei3Pr 2026-02-20T21:28:49.919 DEBUG:teuthology.orchestra.remote:trial196:/tmp/tmp.39xRRei3Pr is 589KB 2026-02-20T21:28:50.018 DEBUG:teuthology.orchestra.run.trial196:> rm -fr /tmp/tmp.39xRRei3Pr 2026-02-20T21:28:50.034 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-20T21:28:50.034 DEBUG:teuthology.orchestra.run.trial196:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-20T21:28:50.076 DEBUG:teuthology.orchestra.run.trial199:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-20T21:28:50.156 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-20T21:28:50.172 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-20T21:28:50.172 INFO:tasks.ceph:Archiving crash dumps... 2026-02-20T21:28:50.175 DEBUG:teuthology.misc:Transferring archived files from trial196:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61415/remote/trial196/crash 2026-02-20T21:28:50.176 DEBUG:teuthology.orchestra.run.trial196:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-20T21:28:50.221 DEBUG:teuthology.misc:Transferring archived files from trial199:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61415/remote/trial199/crash 2026-02-20T21:28:50.222 DEBUG:teuthology.orchestra.run.trial199:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-20T21:28:50.246 INFO:tasks.ceph:Compressing logs... 2026-02-20T21:28:50.246 DEBUG:teuthology.orchestra.run.trial196:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-20T21:28:50.267 DEBUG:teuthology.orchestra.run.trial199:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-20T21:28:50.289 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45811.log 2026-02-20T21:28:50.308 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45781.log 2026-02-20T21:28:50.308 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.45811.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42069.log 2026-02-20T21:28:50.309 INFO:teuthology.orchestra.run.trial196.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45811.log.gz 2026-02-20T21:28:50.309 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.45781.log: gzip 0.0% -5 -- replaced with /var/log/ceph/cluster1-client.admin.45781.log.gz --verbose 2026-02-20T21:28:50.309 INFO:teuthology.orchestra.run.trial196.stderr: -- /var/log/ceph/cluster1-client.admin.45928.log 2026-02-20T21:28:50.309 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.42069.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42069.log.gz 2026-02-20T21:28:50.309 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47841.log 2026-02-20T21:28:50.309 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.45928.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42424.log 2026-02-20T21:28:50.309 INFO:teuthology.orchestra.run.trial196.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45928.log.gz 2026-02-20T21:28:50.309 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.47841.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47841.log.gz 2026-02-20T21:28:50.310 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42203.log 2026-02-20T21:28:50.310 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.42424.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47694.log 2026-02-20T21:28:50.310 INFO:teuthology.orchestra.run.trial196.stderr: 28.9% -- replaced with /var/log/ceph/cluster1-client.admin.42424.log.gz 2026-02-20T21:28:50.310 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.42203.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45314.log 2026-02-20T21:28:50.310 INFO:teuthology.orchestra.run.trial196.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42203.log.gz 2026-02-20T21:28:50.310 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.47694.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47694.log.gz 2026-02-20T21:28:50.310 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-mgr.x.log 2026-02-20T21:28:50.310 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45265.log 2026-02-20T21:28:50.310 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.45314.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45314.log.gz 2026-02-20T21:28:50.310 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster2-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mon.a.log 2026-02-20T21:28:50.311 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.45265.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45265.log.gz 2026-02-20T21:28:50.311 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45200.log 2026-02-20T21:28:50.311 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.2.log 2026-02-20T21:28:50.311 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-mon.a.log: /var/log/ceph/cluster1-client.admin.45200.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45513.log 2026-02-20T21:28:50.311 INFO:teuthology.orchestra.run.trial196.stderr: 80.5% -- replaced with /var/log/ceph/cluster1-client.admin.45200.log.gz 2026-02-20T21:28:50.311 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.0.log 2026-02-20T21:28:50.311 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.45513.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45513.log.gz 2026-02-20T21:28:50.311 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42250.log 2026-02-20T21:28:50.311 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42592.log 2026-02-20T21:28:50.312 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.42250.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42250.log.gz 2026-02-20T21:28:50.312 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47547.log 2026-02-20T21:28:50.312 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.1.log 2026-02-20T21:28:50.312 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.42592.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42592.log.gz 2026-02-20T21:28:50.312 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.47547.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47547.log.gz 2026-02-20T21:28:50.312 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41974.log 2026-02-20T21:28:50.312 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45363.log 2026-02-20T21:28:50.312 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.41974.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45515.log 2026-02-20T21:28:50.312 INFO:teuthology.orchestra.run.trial196.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.41974.log.gz 2026-02-20T21:28:50.313 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.45363.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47645.log 2026-02-20T21:28:50.313 INFO:teuthology.orchestra.run.trial196.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45363.log.gz 2026-02-20T21:28:50.313 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.45515.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47939.log 2026-02-20T21:28:50.313 INFO:teuthology.orchestra.run.trial196.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45515.log.gz 2026-02-20T21:28:50.313 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.47645.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mgr.x.log 2026-02-20T21:28:50.313 INFO:teuthology.orchestra.run.trial196.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47645.log.gz 2026-02-20T21:28:50.313 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.47939.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47939.log.gz 2026-02-20T21:28:50.313 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45782.log 2026-02-20T21:28:50.313 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47449.log 2026-02-20T21:28:50.313 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.45782.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45782.log.gz 2026-02-20T21:28:50.314 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45660.log 2026-02-20T21:28:50.314 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.47449.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45105.log 2026-02-20T21:28:50.314 INFO:teuthology.orchestra.run.trial196.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47449.log.gz 2026-02-20T21:28:50.314 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.45660.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45660.log.gz 2026-02-20T21:28:50.314 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45514.log 2026-02-20T21:28:50.314 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.45105.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45105.log.gz 2026-02-20T21:28:50.314 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42297.log 2026-02-20T21:28:50.314 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.45514.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45514.log.gz 2026-02-20T21:28:50.314 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46147.log 2026-02-20T21:28:50.314 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.42297.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42297.log.gz 2026-02-20T21:28:50.315 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47988.log 2026-02-20T21:28:50.315 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.46147.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46147.log.gz 2026-02-20T21:28:50.315 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.37117.log 2026-02-20T21:28:50.315 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.47988.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47988.log.gz 2026-02-20T21:28:50.315 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45960.log 2026-02-20T21:28:50.315 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/ceph.tmp-client.admin.37117.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.37117.log.gz 2026-02-20T21:28:50.315 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46049.log 2026-02-20T21:28:50.315 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.45960.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45960.log.gz 2026-02-20T21:28:50.315 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.audit.log 2026-02-20T21:28:50.315 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.46049.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46049.log.gzgzip 2026-02-20T21:28:50.315 INFO:teuthology.orchestra.run.trial196.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.admin.46098.log 2026-02-20T21:28:50.316 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45412.log 2026-02-20T21:28:50.316 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.46098.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46098.log.gz 2026-02-20T21:28:50.316 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47743.log 2026-02-20T21:28:50.316 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.45412.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45412.log.gz 2026-02-20T21:28:50.316 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45662.log 2026-02-20T21:28:50.316 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.47743.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47743.log.gz 2026-02-20T21:28:50.316 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47498.log 2026-02-20T21:28:50.316 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.45662.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45662.log.gz 2026-02-20T21:28:50.316 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47792.log 2026-02-20T21:28:50.316 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.47498.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47498.log.gz 2026-02-20T21:28:50.317 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47890.log 2026-02-20T21:28:50.317 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.47792.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47792.log.gz 2026-02-20T21:28:50.317 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45929.log 2026-02-20T21:28:50.317 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.47890.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47890.log.gz 2026-02-20T21:28:50.317 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48035.log 2026-02-20T21:28:50.317 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.45929.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45929.log.gz 2026-02-20T21:28:50.317 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47596.log 2026-02-20T21:28:50.317 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.48035.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48035.log.gz 2026-02-20T21:28:50.317 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45661.log 2026-02-20T21:28:50.317 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.47596.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47596.log.gz 2026-02-20T21:28:50.317 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.log 2026-02-20T21:28:50.318 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1-client.admin.45661.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45175.log 2026-02-20T21:28:50.318 INFO:teuthology.orchestra.run.trial196.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45661.log.gz 2026-02-20T21:28:50.318 INFO:teuthology.orchestra.run.trial196.stderr:/var/log/ceph/cluster1.log: /var/log/ceph/cluster1-client.admin.45175.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45175.log.gz 2026-02-20T21:28:50.318 INFO:teuthology.orchestra.run.trial196.stderr: 87.1% -- replaced with /var/log/ceph/cluster1.log.gz 2026-02-20T21:28:50.318 INFO:teuthology.orchestra.run.trial196.stderr: 93.7% -- replaced with /var/log/ceph/cluster1.audit.log.gz 2026-02-20T21:28:50.346 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56670.log 2026-02-20T21:28:50.346 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74071.log 2026-02-20T21:28:50.347 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.56670.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81643.log 2026-02-20T21:28:50.347 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.56670.log.gz 2026-02-20T21:28:50.347 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.74071.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51122.log 2026-02-20T21:28:50.347 INFO:teuthology.orchestra.run.trial199.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.74071.log.gz 2026-02-20T21:28:50.347 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.81643.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.54624.log 2026-02-20T21:28:50.347 INFO:teuthology.orchestra.run.trial199.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81643.log.gz 2026-02-20T21:28:50.347 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.51122.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89103.log 2026-02-20T21:28:50.347 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.51122.log.gz 2026-02-20T21:28:50.347 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.54624.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.91293.log 2026-02-20T21:28:50.347 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.54624.log.gz 2026-02-20T21:28:50.347 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5/var/log/ceph/cluster2-client.mirror.89103.log: --verbose -- /var/log/ceph/cluster1-client.mirror.69075.log 2026-02-20T21:28:50.347 INFO:teuthology.orchestra.run.trial199.stderr: 80.4% -- replaced with /var/log/ceph/cluster2-client.mirror.89103.log.gz 2026-02-20T21:28:50.347 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.91293.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.91293.log.gz 2026-02-20T21:28:50.347 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85541.log 2026-02-20T21:28:50.347 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.69075.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67198.log 2026-02-20T21:28:50.347 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.69075.log.gz 2026-02-20T21:28:50.347 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.85541.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49610.log 2026-02-20T21:28:50.347 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.85541.log.gz 2026-02-20T21:28:50.347 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.67198.log: gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.37715.log 2026-02-20T21:28:50.347 INFO:teuthology.orchestra.run.trial199.stderr: 87.6%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.67198.log.gz -5 2026-02-20T21:28:50.347 INFO:teuthology.orchestra.run.trial199.stderr: --verbose -- /var/log/ceph/cluster2-client.admin.55323.log 2026-02-20T21:28:50.348 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.49610.log: /var/log/ceph/ceph.tmp-client.admin.37715.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59613.log 2026-02-20T21:28:50.348 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.37715.log.gz 2026-02-20T21:28:50.348 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.55323.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.49610.log.gz 2026-02-20T21:28:50.348 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47117.log 2026-02-20T21:28:50.348 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.55323.log.gz 2026-02-20T21:28:50.348 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.59613.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71136.log 2026-02-20T21:28:50.348 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.47117.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76810.log 2026-02-20T21:28:50.348 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.59613.log.gz -- replaced with /var/log/ceph/cluster1-client.admin.47117.log.gz 2026-02-20T21:28:50.348 INFO:teuthology.orchestra.run.trial199.stderr: 2026-02-20T21:28:50.348 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.71136.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75039.log 2026-02-20T21:28:50.348 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.71136.log.gz 2026-02-20T21:28:50.348 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76810.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49364.log 2026-02-20T21:28:50.348 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75039.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82218.log 2026-02-20T21:28:50.348 INFO:teuthology.orchestra.run.trial199.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76810.log.gz 2026-02-20T21:28:50.348 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.49364.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79373.log 2026-02-20T21:28:50.348 INFO:teuthology.orchestra.run.trial199.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.75039.log.gz 2026-02-20T21:28:50.348 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63208.log 2026-02-20T21:28:50.348 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.82218.log: /var/log/ceph/cluster2-client.mirror.79373.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72020.log 2026-02-20T21:28:50.348 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.82218.log.gz 2026-02-20T21:28:50.348 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.63208.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47019.log 2026-02-20T21:28:50.348 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.63208.log.gz 2026-02-20T21:28:50.348 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.72020.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65142.log 2026-02-20T21:28:50.348 INFO:teuthology.orchestra.run.trial199.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.79373.log.gz 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49364.log.gz 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.47019.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47019.log.gz 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75806.log 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65636.log 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.65142.log: /var/log/ceph/cluster1-client.mirror.75806.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77382.log 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.65142.log.gz 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.75806.log.gz 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.65636.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76314.log 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.77382.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84801.log 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76314.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56124.log 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.77382.log.gz 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.65636.log.gz 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.84801.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43124.log 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.76314.log.gz 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.84801.log.gz 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.56124.log: /var/log/ceph/cluster2-client.admin.43124.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55446.log 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43124.log.gz 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.56124.log.gz 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.80006.log 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.55446.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58570.log 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.55446.log.gz 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.0.80006.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43298.log 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr: 18.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.80006.log.gz 2026-02-20T21:28:50.349 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.58570.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81666.log 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.43298.log: gzip -5 --verbose 87.4% -- /var/log/ceph/cluster2-client.mirror.55010.log -- replaced with /var/log/ceph/cluster1-client.admin.58570.log.gz 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr: 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43298.log.gz 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.72020.log.gz 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.81666.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55521.log 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.81666.log.gz 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.55010.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59640.log 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.55521.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53961.log 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.55010.log.gz 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.55521.log.gz 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.59640.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60384.log 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.53961.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.59640.log.gz 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81554.log 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.60384.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67222.log 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.60384.log.gz/var/log/ceph/cluster1-client.mirror.81554.log: 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52668.log 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.81554.log.gz 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.67222.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66701.log 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.52668.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51496.log 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.67222.log.gz 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.66701.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.admin.52668.log.gz 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56697.log 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.51496.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.66701.log.gz 2026-02-20T21:28:50.350 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50267.log 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.51496.log.gz 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.56697.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46823.log 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.56697.log.gz 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.50267.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54759.log 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.50267.log.gz 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.46823.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62714.log 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46823.log.gz 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.54759.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54032.log 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.62714.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77733.log 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr: 87.3%/var/log/ceph/cluster1-client.mirror.54032.log: -- replaced with /var/log/ceph/cluster1-client.admin.62714.log.gz 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53961.log.gz 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51170.log 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54032.log.gz 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.77733.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53100.log 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.51170.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89683.log 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.51170.log.gz 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.53100.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65497.log 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77733.log.gz 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.89683.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.53100.log.gz 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49301.log 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.65497.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74610.log 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.65497.log.gz 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.49301.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63801.log 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.74610.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54056.log 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.63801.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74242.log 2026-02-20T21:28:50.351 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63801.log.gz 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.74610.log.gz 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.54056.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.80055.log 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.49301.log.gz 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.74242.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70542.log 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54056.log.gz 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr: 94.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54759.log.gz 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.2.80055.log: 19.1% -- replaced with /var/log/ceph/cluster1-client.mirror.2.80055.log.gz 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82999.log 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.74242.log.gz 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78233.log 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.70542.log: /var/log/ceph/cluster1-client.mirror.82999.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57445.log 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.78233.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77020.log 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.70542.log.gz 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.78233.log.gz 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.57445.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.80476.log 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.57445.log.gz 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.77020.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.82999.log.gz 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72095.log 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77020.log.gz 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.80476.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68541.log 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.72095.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.80476.log.gz 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79922.log 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.68541.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65017.log 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr: 95.0% -- replaced with /var/log/ceph/cluster1-client.mirror.89683.log.gz 2026-02-20T21:28:50.352 INFO:teuthology.orchestra.run.trial199.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.72095.log.gz 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.68541.log.gz 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.79922.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72000.log 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.65017.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70390.log 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.65017.log.gz 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.72000.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57651.log 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72000.log.gz 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.70390.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48071.log 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.57651.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.88925.log 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.70390.log.gz 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.57651.log.gz 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.48071.log: 56.5% -- replaced with /var/log/ceph/cluster1-client.admin.48071.log.gz 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71302.log 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.88925.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81927.log 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.71302.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.88925.log.gz 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68793.log 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.81927.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52558.log 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.71302.log.gz 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.68793.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68175.log 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.68793.log.gz 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.52558.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85384.log 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.52558.log.gz 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.68175.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74094.log 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.68175.log.gz 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.85384.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73333.log 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.85384.log.gz 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.74094.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.74094.log.gz 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59711.log 2026-02-20T21:28:50.353 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.73333.log: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.79922.log.gz 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2.log 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.59711.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.73333.log.gz 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53073.log 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.59711.log.gz 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76831.log 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.53073.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72966.log 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76831.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.53073.log.gz 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.80617.log 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76831.log.gz 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.72966.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67401.log 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.72966.log.gz 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.80617.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52034.log 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.80617.log.gz 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.67401.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84257.log 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81927.log.gz 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.67401.log.gz 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.52034.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67481.log 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.84257.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59870.log 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr: 90.6% -- replaced with /var/log/ceph/cluster1-client.admin.52034.log.gz 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.84257.log.gz 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.67481.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71963.log 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr: 86.9% -- replaced with /var/log/ceph/cluster2.log.gz 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.59870.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70516.log 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.71963.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49387.log 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr: 80.3% -- replaced with /var/log/ceph/cluster2-client.mirror.71963.log.gz 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67481.log.gz 2026-02-20T21:28:50.354 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.70516.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65943.log 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.59870.log.gz 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.70516.log.gz 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.49387.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78208.log 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr: 86.5% -- replaced with /var/log/ceph/cluster1-client.mirror.49387.log.gz 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51146.log 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.65943.log: /var/log/ceph/cluster1-client.mirror.78208.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56231.log 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.65943.log.gz 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78208.log.gz 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.51146.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66138.log 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.51146.log.gz 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.56231.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86945.log 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.66138.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.56231.log.gz 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53476.log 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.66138.log.gz 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.86945.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75612.log 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.86945.log.gz 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.53476.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75447.log 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75612.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52106.log 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75447.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.75612.log.gz 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90447.log 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53476.log.gz 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.52106.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52439.log 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.52106.log.gz 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.90447.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46137.log 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70610.log 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.52439.log: /var/log/ceph/cluster2-client.admin.46137.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46137.log.gz 2026-02-20T21:28:50.355 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53728.log 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69337.log 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.70610.log: 90.5% -- replaced with /var/log/ceph/cluster1-client.admin.52439.log.gz 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.53728.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.53728.log.gz 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73437.log 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82616.log 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.69337.log: /var/log/ceph/cluster2-client.mirror.73437.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.70610.log.gz 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.81108.log 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.69337.log.gz 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.82616.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65764.log 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.73437.log.gz 4.7% 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.82616.log.gz 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.0.81108.log: 19.7% -- replaced with /var/log/ceph/cluster1-client.mirror.0.81108.log.gz 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53195.log 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.65764.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49539.log 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.53195.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.65764.log.gz 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.90235.log 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.49539.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53195.log.gz 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84032.log 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49539.log.gz 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85611.log 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.90235.log: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75447.log.gz 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.84032.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51576.log 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.90235.log.gz 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.85611.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62287.log 2026-02-20T21:28:50.356 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.85611.log.gz 2026-02-20T21:28:50.363 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.51576.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65043.log 2026-02-20T21:28:50.363 INFO:teuthology.orchestra.run.trial199.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.84032.log.gz 2026-02-20T21:28:50.363 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.62287.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49410.log 2026-02-20T21:28:50.363 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.62287.log.gz 2026-02-20T21:28:50.363 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.65043.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57705.log 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr: 95.1% -- replaced with /var/log/ceph/cluster2-client.mirror.90447.log.gz 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.49410.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89510.log 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.51576.log.gz 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65043.log.gz 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr: 86.5% -- replaced with /var/log/ceph/cluster1-client.mirror.49410.log.gz 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.57705.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.77504.log 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.57705.log.gz 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.89510.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50171.log 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.77504.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72678.log 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.77504.log.gz 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.50171.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.84921.log 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.72678.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72990.log 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.3.84921.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57190.log 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr: 19.7% -- replaced with /var/log/ceph/cluster1-client.mirror.3.84921.log.gz 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.72990.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85906.log 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.72990.log.gz 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.57190.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71195.log 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr: 87.7%/var/log/ceph/cluster1-client.admin.85906.log: -- replaced with /var/log/ceph/cluster2-client.mirror.57190.log.gz 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56363.log 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.85906.log.gz 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr:gzip/var/log/ceph/cluster1-client.admin.71195.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72574.log 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.56363.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.71195.log.gz 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62118.log 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.56363.log.gz 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.72574.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49241.log 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.72574.log.gz 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.62118.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.80548.log 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.49241.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73805.log 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.49241.log.gz 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.80548.log: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.62118.log.gz 2026-02-20T21:28:50.364 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59399.log 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.73805.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.80548.log.gz 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59265.log 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.59399.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77103.log 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.59265.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.59399.log.gz 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72918.log 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.73805.log.gz 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.59265.log.gz 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.77103.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.54881.log 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.50171.log.gz 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.77103.log.gz 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.72918.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67704.log 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.72918.log.gz 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.54881.log: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.72678.log.gz 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54908.log 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.54881.log.gz 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.67704.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66813.log 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.54908.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.67704.log.gz 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88551.log 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.54908.log.gz 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.66813.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53803.log 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66813.log.gz 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.88551.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69937.log 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.88551.log.gz 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.53803.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.50692.log 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.53803.log.gz 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.69937.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80839.log 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.69937.log.gz 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.1.50692.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88635.log 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr: 19.1% -- replaced with /var/log/ceph/cluster1-client.mirror.1.50692.log.gz 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.80839.log: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.80839.log.gz 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55232.log 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.88635.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68478.log 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.88635.log.gz 2026-02-20T21:28:50.365 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.55232.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67303.log 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.68478.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76467.log 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.67303.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53279.log 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% 90.6% -- replaced with /var/log/ceph/cluster2-client.admin.68478.log.gz 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.55232.log.gz 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.76467.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.67303.log.gz 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66063.log 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.76467.log.gz 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.53279.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75828.log 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.53279.log.gz 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.66063.log: -- /var/log/ceph/cluster2-client.mirror.74302.log 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.75828.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48208.log 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66063.log.gz 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.75828.log.gz 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.74302.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83616.log 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.48208.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.74302.log.gz 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr: 79.9% -- replaced with /var/log/ceph/cluster2-client.admin.48208.log.gz 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89469.log 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83376.log 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.83616.log: /var/log/ceph/cluster2-client.mirror.89469.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.83616.log.gz 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74589.log 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.83376.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49217.log 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.83376.log.gz 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.74589.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53633.log 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.74589.log.gz 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.49217.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61464.log 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.49217.log.gz 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.53633.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78981.log 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53633.log.gz 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.61464.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59016.log 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.78981.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.61464.log.gz 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65862.log 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.59016.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.admin.78981.log.gz 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54703.log 2026-02-20T21:28:50.366 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.59016.log.gz 2026-02-20T21:28:50.367 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.65862.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56643.log 2026-02-20T21:28:50.367 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.65862.log.gz 2026-02-20T21:28:50.367 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.54703.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85766.log 2026-02-20T21:28:50.367 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.56643.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73745.log 2026-02-20T21:28:50.367 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.56643.log.gz 2026-02-20T21:28:50.367 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.85766.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64605.log 2026-02-20T21:28:50.367 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54703.log.gz 2026-02-20T21:28:50.367 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.85766.log.gz 2026-02-20T21:28:50.367 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.73745.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58677.log 2026-02-20T21:28:50.367 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.64605.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89552.log 2026-02-20T21:28:50.367 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.73745.log.gz -- replaced with /var/log/ceph/cluster2-client.admin.64605.log.gz 2026-02-20T21:28:50.367 INFO:teuthology.orchestra.run.trial199.stderr: 2026-02-20T21:28:50.367 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.58677.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49835.log 2026-02-20T21:28:50.369 INFO:teuthology.orchestra.run.trial199.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89469.log.gz 2026-02-20T21:28:50.369 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.89552.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60563.log 2026-02-20T21:28:50.369 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58677.log.gz 2026-02-20T21:28:50.369 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.49835.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59998.log 2026-02-20T21:28:50.369 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.89552.log.gz 2026-02-20T21:28:50.369 INFO:teuthology.orchestra.run.trial199.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.49835.log.gz 2026-02-20T21:28:50.369 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.60563.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52133.log 2026-02-20T21:28:50.369 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.60563.log.gz 2026-02-20T21:28:50.369 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.59998.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69283.log 2026-02-20T21:28:50.369 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.59998.log.gz 2026-02-20T21:28:50.369 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.52133.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64334.log 2026-02-20T21:28:50.369 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.69283.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59666.log 2026-02-20T21:28:50.369 INFO:teuthology.orchestra.run.trial199.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.52133.log.gz 87.7% 2026-02-20T21:28:50.369 INFO:teuthology.orchestra.run.trial199.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.69283.log.gz 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.64334.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70283.log 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.64334.log.gz 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.59666.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48681.log 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.admin.70283.log: /var/log/ceph/cluster1-client.mirror.84653.log 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.48681.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59666.log.gzgzip 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.admin.58810.log 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.70283.log.gz 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr: 55.8% -- replaced with /var/log/ceph/cluster1-client.mirror.48681.log.gz 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.84653.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85480.log 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.58810.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60357.log 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.58810.log.gz 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.85480.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48259.log 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr: 91.3% -- replaced with /var/log/ceph/cluster1-client.mirror.84653.log.gz 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.60357.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53150.log 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.60357.log.gz 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.48259.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83424.log 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48259.log.gz 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.53150.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55650.log 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.83424.log: gzip 88.4% -5 --verbose -- replaced with /var/log/ceph/cluster2-client.mirror.53150.log.gz -- 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr: /var/log/ceph/cluster2-client.mirror.2.86126.log 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.85480.log.gz 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.83424.log.gz 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.55650.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66165.log 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.2.86126.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68023.log 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr: 19.1% -- replaced with /var/log/ceph/cluster2-client.mirror.2.86126.log.gz 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.66165.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.55650.log.gz 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54806.log 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.66165.log.gz 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.68023.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66456.log 2026-02-20T21:28:50.370 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.68023.log.gz 2026-02-20T21:28:50.371 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.54806.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.78521.log 2026-02-20T21:28:50.371 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54806.log.gz 2026-02-20T21:28:50.371 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.66456.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70979.log 2026-02-20T21:28:50.371 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.66456.log.gz 2026-02-20T21:28:50.371 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.78521.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53559.log 2026-02-20T21:28:50.371 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.70979.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67148.log 2026-02-20T21:28:50.371 INFO:teuthology.orchestra.run.trial199.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.78521.log.gz 2026-02-20T21:28:50.372 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.53559.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.89757.log 2026-02-20T21:28:50.372 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.67148.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.77808.log 2026-02-20T21:28:50.372 INFO:teuthology.orchestra.run.trial199.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53559.log.gz 2026-02-20T21:28:50.372 INFO:teuthology.orchestra.run.trial199.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.70979.log.gz 2026-02-20T21:28:50.372 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63852.log 2026-02-20T21:28:50.372 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.1.89757.log: /var/log/ceph/cluster1-client.mirror.2.77808.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72759.log 2026-02-20T21:28:50.372 INFO:teuthology.orchestra.run.trial199.stderr: 90.4% 19.1% -- replaced with /var/log/ceph/cluster2-client.mirror.67148.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.2.77808.log.gz 2026-02-20T21:28:50.372 INFO:teuthology.orchestra.run.trial199.stderr: 2026-02-20T21:28:50.372 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.63852.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57132.log 2026-02-20T21:28:50.372 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.63852.log.gz 2026-02-20T21:28:50.372 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.72759.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67024.log 2026-02-20T21:28:50.372 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.57132.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62501.log 2026-02-20T21:28:50.372 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.67024.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73134.log 2026-02-20T21:28:50.372 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.67024.log.gz 2026-02-20T21:28:50.372 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.62501.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.57132.log.gz 2026-02-20T21:28:50.373 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71412.log 2026-02-20T21:28:50.373 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.62501.log.gz 2026-02-20T21:28:50.373 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.73134.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.80384.log 2026-02-20T21:28:50.373 INFO:teuthology.orchestra.run.trial199.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.73134.log.gz 2026-02-20T21:28:50.373 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.71412.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76535.log 2026-02-20T21:28:50.373 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.80384.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.71412.log.gz 2026-02-20T21:28:50.373 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65817.log 2026-02-20T21:28:50.373 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.80384.log.gz 2026-02-20T21:28:50.373 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.76535.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57811.log 2026-02-20T21:28:50.373 INFO:teuthology.orchestra.run.trial199.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.76535.log.gz 2026-02-20T21:28:50.374 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.65817.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.54190.log 2026-02-20T21:28:50.374 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.57811.log: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.1.89757.log.gz 2026-02-20T21:28:50.374 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72552.log 2026-02-20T21:28:50.374 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.57811.log.gz 2026-02-20T21:28:50.374 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65817.log.gz 2026-02-20T21:28:50.374 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.1.54190.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68148.log 2026-02-20T21:28:50.375 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.72552.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71626.log 2026-02-20T21:28:50.375 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67810.log 2026-02-20T21:28:50.375 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.68148.log: /var/log/ceph/cluster2-client.admin.71626.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.68148.log.gzgzip 2026-02-20T21:28:50.375 INFO:teuthology.orchestra.run.trial199.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76621.log 2026-02-20T21:28:50.375 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.71626.log.gz 2026-02-20T21:28:50.375 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.67810.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67549.log 2026-02-20T21:28:50.375 INFO:teuthology.orchestra.run.trial199.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.72552.log.gz 2026-02-20T21:28:50.376 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76621.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.67810.log.gz 2026-02-20T21:28:50.376 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86823.log 2026-02-20T21:28:50.376 INFO:teuthology.orchestra.run.trial199.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76621.log.gz 2026-02-20T21:28:50.376 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.67549.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54935.log 2026-02-20T21:28:50.376 INFO:teuthology.orchestra.run.trial199.stderr: 93.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72759.log.gz 2026-02-20T21:28:50.376 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.86823.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51854.log 2026-02-20T21:28:50.376 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.54935.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86650.log 2026-02-20T21:28:50.377 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.51854.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.86823.log.gz 2026-02-20T21:28:50.377 INFO:teuthology.orchestra.run.trial199.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.67549.log.gz 2026-02-20T21:28:50.377 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.54935.log.gz 2026-02-20T21:28:50.377 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58864.log 2026-02-20T21:28:50.377 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.51854.log.gz 2026-02-20T21:28:50.377 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.86650.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85460.log 2026-02-20T21:28:50.377 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.86650.log.gz 2026-02-20T21:28:50.377 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.58864.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89046.log 2026-02-20T21:28:50.377 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.58864.log.gz 2026-02-20T21:28:50.377 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.85460.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46774.log 2026-02-20T21:28:50.378 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84143.log 2026-02-20T21:28:50.378 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.89046.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.85460.log.gz 2026-02-20T21:28:50.378 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.46774.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63654.log 2026-02-20T21:28:50.378 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46774.log.gz 2026-02-20T21:28:50.378 INFO:teuthology.orchestra.run.trial199.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.89046.log.gz 2026-02-20T21:28:50.378 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.84143.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57472.log 2026-02-20T21:28:50.378 INFO:teuthology.orchestra.run.trial199.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.84143.log.gz 2026-02-20T21:28:50.378 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.63654.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82095.log 2026-02-20T21:28:50.379 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.63654.log.gz 2026-02-20T21:28:50.379 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.57472.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82979.log 2026-02-20T21:28:50.379 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.57472.log.gz 2026-02-20T21:28:50.379 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.82095.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62633.log 2026-02-20T21:28:50.379 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.82095.log.gz 2026-02-20T21:28:50.379 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.82979.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63969.log 2026-02-20T21:28:50.379 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.62633.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70462.log 2026-02-20T21:28:50.379 INFO:teuthology.orchestra.run.trial199.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.62633.log.gz 2026-02-20T21:28:50.379 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74865.log 2026-02-20T21:28:50.380 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.63969.log: /var/log/ceph/cluster1-client.admin.70462.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69562.log 2026-02-20T21:28:50.380 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.63969.log.gz 2026-02-20T21:28:50.380 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.70462.log.gz 2026-02-20T21:28:50.380 INFO:teuthology.orchestra.run.trial199.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82979.log.gz 2026-02-20T21:28:50.380 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.74865.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68718.log 2026-02-20T21:28:50.380 INFO:teuthology.orchestra.run.trial199.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.74865.log.gz 2026-02-20T21:28:50.380 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.69562.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77712.log 2026-02-20T21:28:50.381 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.68718.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77480.log 2026-02-20T21:28:50.381 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.69562.log.gz 2026-02-20T21:28:50.381 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.68718.log.gz 2026-02-20T21:28:50.381 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.77712.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.80015.log 2026-02-20T21:28:50.381 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.77480.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66309.log 2026-02-20T21:28:50.381 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.77480.log.gz 2026-02-20T21:28:50.381 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71980.log 2026-02-20T21:28:50.381 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.0.80015.log: /var/log/ceph/cluster2-client.admin.66309.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.89416.log 2026-02-20T21:28:50.381 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.66309.log.gz 2026-02-20T21:28:50.382 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.71980.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48984.log 2026-02-20T21:28:50.382 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.89416.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.79795.log 2026-02-20T21:28:50.382 INFO:teuthology.orchestra.run.trial199.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.71980.log.gz 2026-02-20T21:28:50.382 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.48984.log: gzip -5 --verbose 88.5% -- -- replaced with /var/log/ceph/cluster2-client.admin.89416.log.gz /var/log/ceph/cluster2-client.mirror.73050.log 2026-02-20T21:28:50.382 INFO:teuthology.orchestra.run.trial199.stderr: 2026-02-20T21:28:50.382 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.79795.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49855.log 2026-02-20T21:28:50.383 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.73050.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48984.log.gz 2026-02-20T21:28:50.383 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56817.log 2026-02-20T21:28:50.383 INFO:teuthology.orchestra.run.trial199.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.admin.79795.log.gz 2026-02-20T21:28:50.383 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.49855.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47896.log 2026-02-20T21:28:50.383 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.56817.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49321.log 2026-02-20T21:28:50.384 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.56817.log.gz 2026-02-20T21:28:50.384 INFO:teuthology.orchestra.run.trial199.stderr:gzip/var/log/ceph/cluster1-client.mirror.47896.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76355.log 2026-02-20T21:28:50.384 INFO:teuthology.orchestra.run.trial199.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.73050.log.gz 2026-02-20T21:28:50.384 INFO:teuthology.orchestra.run.trial199.stderr: 62.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47896.log.gz 2026-02-20T21:28:50.384 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.49321.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59792.log 2026-02-20T21:28:50.384 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63502.log 2026-02-20T21:28:50.384 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76355.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49855.log.gz 2026-02-20T21:28:50.384 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.59792.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65115.log 2026-02-20T21:28:50.384 INFO:teuthology.orchestra.run.trial199.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77712.log.gz 2026-02-20T21:28:50.385 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.63502.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.59792.log.gz 2026-02-20T21:28:50.385 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46436.log 2026-02-20T21:28:50.385 INFO:teuthology.orchestra.run.trial199.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.49321.log.gz 2026-02-20T21:28:50.385 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.65115.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.89812.log 2026-02-20T21:28:50.385 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.63502.log.gz 2026-02-20T21:28:50.385 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.65115.log.gz 2026-02-20T21:28:50.385 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.46436.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.88116.log 2026-02-20T21:28:50.385 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46436.log.gz 2026-02-20T21:28:50.385 INFO:teuthology.orchestra.run.trial199.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.76355.log.gz 2026-02-20T21:28:50.385 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.2.89812.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63129.log 2026-02-20T21:28:50.385 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.3.88116.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82946.log 2026-02-20T21:28:50.385 INFO:teuthology.orchestra.run.trial199.stderr: 19.7% -- replaced with /var/log/ceph/cluster1-client.mirror.3.88116.log.gz 2026-02-20T21:28:50.386 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.63129.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.0.80015.log.gz 2026-02-20T21:28:50.386 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71005.log 2026-02-20T21:28:50.386 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.63129.log.gz 2026-02-20T21:28:50.386 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.82946.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52947.log 2026-02-20T21:28:50.386 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.82946.log.gz 2026-02-20T21:28:50.386 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.71005.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82374.log 2026-02-20T21:28:50.386 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.52947.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.71005.log.gz 2026-02-20T21:28:50.386 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55891.log 2026-02-20T21:28:50.386 INFO:teuthology.orchestra.run.trial199.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.2.89812.log.gz 2026-02-20T21:28:50.387 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.82374.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.52947.log.gz 2026-02-20T21:28:50.387 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73638.log 2026-02-20T21:28:50.387 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.55891.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46039.log 2026-02-20T21:28:50.387 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.73638.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83571.log 2026-02-20T21:28:50.388 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.46039.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.73638.log.gz 2026-02-20T21:28:50.388 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73765.log 2026-02-20T21:28:50.388 INFO:teuthology.orchestra.run.trial199.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.82374.log.gz 2026-02-20T21:28:50.388 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46039.log.gz 2026-02-20T21:28:50.388 INFO:teuthology.orchestra.run.trial199.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55891.log.gz 2026-02-20T21:28:50.388 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.83571.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51523.log 2026-02-20T21:28:50.388 INFO:teuthology.orchestra.run.trial196.stderr: 91.1% -- replaced with /var/log/ceph/cluster2-mgr.x.log.gz 2026-02-20T21:28:50.388 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.73765.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56865.log 2026-02-20T21:28:50.388 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.51523.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84448.log 2026-02-20T21:28:50.388 INFO:teuthology.orchestra.run.trial199.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.73765.log.gz 2026-02-20T21:28:50.389 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.56865.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.51523.log.gz 2026-02-20T21:28:50.389 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.77558.log 2026-02-20T21:28:50.389 INFO:teuthology.orchestra.run.trial199.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.56865.log.gz 2026-02-20T21:28:50.389 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.84448.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75140.log 2026-02-20T21:28:50.389 INFO:teuthology.orchestra.run.trial199.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.83571.log.gz 2026-02-20T21:28:50.389 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.77558.log: 87.0%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.84448.log.gz -5 2026-02-20T21:28:50.389 INFO:teuthology.orchestra.run.trial199.stderr: --verbose -- /var/log/ceph/cluster2-client.admin.58783.log 2026-02-20T21:28:50.389 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75140.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.admin.77558.log.gz 2026-02-20T21:28:50.389 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82823.log 2026-02-20T21:28:50.389 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.58783.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81854.log 2026-02-20T21:28:50.390 INFO:teuthology.orchestra.run.trial199.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.58783.log.gz 2026-02-20T21:28:50.390 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.82823.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71029.log 2026-02-20T21:28:50.390 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.82823.log.gz 2026-02-20T21:28:50.390 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.81854.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50043.log 2026-02-20T21:28:50.390 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81854.log.gz 2026-02-20T21:28:50.390 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.71029.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58651.log 2026-02-20T21:28:50.390 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.71029.log.gz 2026-02-20T21:28:50.390 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.50043.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46287.log 2026-02-20T21:28:50.391 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.58651.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86399.log 2026-02-20T21:28:50.391 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50043.log.gz 2026-02-20T21:28:50.391 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.58651.log.gz 2026-02-20T21:28:50.391 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.46287.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46287.log.gz 2026-02-20T21:28:50.391 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48916.log 2026-02-20T21:28:50.391 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82650.log 2026-02-20T21:28:50.391 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.86399.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.86399.log.gz 2026-02-20T21:28:50.391 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.48916.log: 58.0% -- replaced with /var/log/ceph/cluster2-client.mirror.48916.log.gz 2026-02-20T21:28:50.392 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83691.log 2026-02-20T21:28:50.392 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.82650.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.85007.log 2026-02-20T21:28:50.392 INFO:teuthology.orchestra.run.trial199.stderr: 69.6% -- replaced with /var/log/ceph/cluster2-client.mirror.82650.log.gz 2026-02-20T21:28:50.392 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.83691.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56545.log 2026-02-20T21:28:50.392 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.3.85007.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66362.log 2026-02-20T21:28:50.392 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.56545.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.83691.log.gz 2026-02-20T21:28:50.392 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66837.log 2026-02-20T21:28:50.393 INFO:teuthology.orchestra.run.trial199.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75140.log.gz 2026-02-20T21:28:50.393 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.66362.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49193.log 2026-02-20T21:28:50.393 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.66837.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.84870.log 2026-02-20T21:28:50.393 INFO:teuthology.orchestra.run.trial199.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.66837.log.gz 2026-02-20T21:28:50.393 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.49193.log: 89.7%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.66362.log.gz -5 2026-02-20T21:28:50.393 INFO:teuthology.orchestra.run.trial199.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.63397.log 2026-02-20T21:28:50.394 INFO:teuthology.orchestra.run.trial199.stderr: 91.3% -- replaced with /var/log/ceph/cluster1-client.mirror.56545.log.gz 2026-02-20T21:28:50.394 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.49193.log.gz 2026-02-20T21:28:50.394 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.2.84870.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73985.log 2026-02-20T21:28:50.394 INFO:teuthology.orchestra.run.trial199.stderr: 18.1% -- replaced with /var/log/ceph/cluster1-client.mirror.2.84870.log.gz 2026-02-20T21:28:50.394 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.63397.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69660.log 2026-02-20T21:28:50.394 INFO:teuthology.orchestra.run.trial199.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.3.85007.log.gz 2026-02-20T21:28:50.394 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.63397.log.gz 2026-02-20T21:28:50.394 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.73985.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76705.log 2026-02-20T21:28:50.394 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.73985.log.gz 2026-02-20T21:28:50.394 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose --/var/log/ceph/cluster1-client.admin.69660.log: /var/log/ceph/cluster1-client.admin.54570.log 2026-02-20T21:28:50.395 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76705.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78703.log 2026-02-20T21:28:50.395 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.69660.log.gz 2026-02-20T21:28:50.395 INFO:teuthology.orchestra.run.trial199.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76705.log.gz 2026-02-20T21:28:50.395 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.54570.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75100.log 2026-02-20T21:28:50.395 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.54570.log.gz 2026-02-20T21:28:50.395 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.78703.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47844.log 2026-02-20T21:28:50.395 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75100.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47861.log 2026-02-20T21:28:50.396 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.47844.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73257.log 2026-02-20T21:28:50.396 INFO:teuthology.orchestra.run.trial199.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.75100.log.gz 2026-02-20T21:28:50.396 INFO:teuthology.orchestra.run.trial199.stderr: 79.8% -- replaced with /var/log/ceph/cluster1-client.admin.47844.log.gz 2026-02-20T21:28:50.396 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.47861.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46592.log 2026-02-20T21:28:50.396 INFO:teuthology.orchestra.run.trial199.stderr: 80.0% -- replaced with /var/log/ceph/cluster1-client.admin.47861.log.gz 2026-02-20T21:28:50.396 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.73257.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55968.log 2026-02-20T21:28:50.396 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.46592.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46592.log.gz 2026-02-20T21:28:50.396 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83551.log 2026-02-20T21:28:50.397 INFO:teuthology.orchestra.run.trial199.stderr: 90.8% -- replaced with /var/log/ceph/cluster1-client.admin.73257.log.gz 2026-02-20T21:28:50.397 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.55968.log: gzip -5 --verbose -- /var/log/ceph/cluster2.audit.log 2026-02-20T21:28:50.397 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.83551.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.55968.log.gz 2026-02-20T21:28:50.397 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55802.log 2026-02-20T21:28:50.397 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2.audit.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83551.log.gz 2026-02-20T21:28:50.397 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67837.log 2026-02-20T21:28:50.397 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.55802.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78184.log 2026-02-20T21:28:50.398 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.67837.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.50864.log 2026-02-20T21:28:50.398 INFO:teuthology.orchestra.run.trial199.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55802.log.gz 2026-02-20T21:28:50.398 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.67837.log.gz 2026-02-20T21:28:50.398 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.78184.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70337.log 2026-02-20T21:28:50.398 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.78184.log.gz 2026-02-20T21:28:50.398 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.3.50864.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61966.log 2026-02-20T21:28:50.399 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.70337.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79642.log 2026-02-20T21:28:50.399 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.70337.log.gz 2026-02-20T21:28:50.399 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.61966.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88871.log 2026-02-20T21:28:50.399 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.79642.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52259.log 2026-02-20T21:28:50.399 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.79642.log.gz 2026-02-20T21:28:50.399 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.88871.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.61966.log.gz 2026-02-20T21:28:50.399 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51096.log 2026-02-20T21:28:50.400 INFO:teuthology.orchestra.run.trial199.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.78703.log.gz 2026-02-20T21:28:50.400 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.52259.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85698.log 2026-02-20T21:28:50.400 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.52259.log.gz 2026-02-20T21:28:50.400 INFO:teuthology.orchestra.run.trial199.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.88871.log.gz 2026-02-20T21:28:50.400 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.51096.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46289.log 2026-02-20T21:28:50.400 INFO:teuthology.orchestra.run.trial196.stderr: 90.9% -- replaced with /var/log/ceph/cluster1-mgr.x.log.gz 2026-02-20T21:28:50.400 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.85698.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77082.log 2026-02-20T21:28:50.400 INFO:teuthology.orchestra.run.trial199.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51096.log.gz 2026-02-20T21:28:50.401 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.85698.log.gz 2026-02-20T21:28:50.401 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.46289.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46289.log.gz 2026-02-20T21:28:50.401 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.84832.log 2026-02-20T21:28:50.401 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.77082.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53395.log 2026-02-20T21:28:50.401 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.77082.log.gz 2026-02-20T21:28:50.401 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.1.84832.log: 19.1% -- replaced with /var/log/ceph/cluster1-client.mirror.1.84832.log.gz 2026-02-20T21:28:50.401 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84355.log 2026-02-20T21:28:50.401 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.53395.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.54295.log 2026-02-20T21:28:50.401 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.84355.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53395.log.gz 2026-02-20T21:28:50.401 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72638.log 2026-02-20T21:28:50.401 INFO:teuthology.orchestra.run.trial199.stderr: 93.7% -- replaced with /var/log/ceph/cluster2.audit.log.gz 2026-02-20T21:28:50.402 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.3.54295.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57731.log 2026-02-20T21:28:50.402 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.72638.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57961.log 2026-02-20T21:28:50.402 INFO:teuthology.orchestra.run.trial199.stderr: 91.2% -- replaced with /var/log/ceph/cluster2-client.mirror.84355.log.gz 2026-02-20T21:28:50.402 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.57731.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69310.log 2026-02-20T21:28:50.402 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72638.log.gz 2026-02-20T21:28:50.403 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.57961.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83475.log 2026-02-20T21:28:50.403 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.57731.log.gz 2026-02-20T21:28:50.403 INFO:teuthology.orchestra.run.trial199.stderr: 87.7%/var/log/ceph/cluster1-client.admin.69310.log: -- replaced with /var/log/ceph/cluster2-client.mirror.57961.log.gz 2026-02-20T21:28:50.403 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72324.log 2026-02-20T21:28:50.403 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.69310.log.gz 2026-02-20T21:28:50.403 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.83475.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68229.log 2026-02-20T21:28:50.403 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.83475.log.gz 2026-02-20T21:28:50.403 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.72324.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57214.log 2026-02-20T21:28:50.403 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.68229.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.50796.log 2026-02-20T21:28:50.403 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.68229.log.gz 2026-02-20T21:28:50.404 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.57214.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82245.log 2026-02-20T21:28:50.404 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.57214.log.gz 2026-02-20T21:28:50.404 INFO:teuthology.orchestra.run.trial199.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.72324.log.gz 2026-02-20T21:28:50.404 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.2.50796.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58624.log 2026-02-20T21:28:50.404 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.82245.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52187.log 2026-02-20T21:28:50.404 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.82245.log.gz 2026-02-20T21:28:50.404 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.58624.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.88012.log 2026-02-20T21:28:50.405 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.58624.log.gz 2026-02-20T21:28:50.405 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.52187.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.88010.log 2026-02-20T21:28:50.405 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.1.88012.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.52187.log.gz 2026-02-20T21:28:50.405 INFO:teuthology.orchestra.run.trial199.stderr: 19.1% -- replaced with /var/log/ceph/cluster1-client.mirror.1.88012.log.gz 2026-02-20T21:28:50.405 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83788.log 2026-02-20T21:28:50.405 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.0.88010.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62065.log 2026-02-20T21:28:50.405 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.83788.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84600.log 2026-02-20T21:28:50.406 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.83788.log.gz 2026-02-20T21:28:50.406 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.62065.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73722.log 2026-02-20T21:28:50.406 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.62065.log.gz 2026-02-20T21:28:50.406 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.84600.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46088.log 2026-02-20T21:28:50.406 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.84600.log.gz 2026-02-20T21:28:50.406 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.73722.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84055.log 2026-02-20T21:28:50.406 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.46088.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46088.log.gz 2026-02-20T21:28:50.407 INFO:teuthology.orchestra.run.trial199.stderr:gzip 89.6% -5 --verbose -- replaced with /var/log/ceph/cluster2-client.mirror.73722.log.gz -- 2026-02-20T21:28:50.407 INFO:teuthology.orchestra.run.trial199.stderr: /var/log/ceph/cluster1-client.mirror.89066.log 2026-02-20T21:28:50.407 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.84055.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49732.log 2026-02-20T21:28:50.407 INFO:teuthology.orchestra.run.trial199.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.84055.log.gz 2026-02-20T21:28:50.407 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.89066.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73017.log 2026-02-20T21:28:50.407 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.49732.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67783.log 2026-02-20T21:28:50.407 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.49732.log.gz 2026-02-20T21:28:50.408 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.73017.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61159.log 2026-02-20T21:28:50.408 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.67783.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.73017.log.gz 2026-02-20T21:28:50.408 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55494.log 2026-02-20T21:28:50.408 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.67783.log.gz 2026-02-20T21:28:50.408 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.61159.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82485.log 2026-02-20T21:28:50.408 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.61159.log.gz 2026-02-20T21:28:50.408 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.55494.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62212.log 2026-02-20T21:28:50.408 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.55494.log.gz 2026-02-20T21:28:50.408 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.82485.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81760.log 2026-02-20T21:28:50.409 INFO:teuthology.orchestra.run.trial199.stderr: 86.4% -- replaced with /var/log/ceph/cluster1-client.mirror.82485.log.gz 2026-02-20T21:28:50.409 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.62212.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48088.log 2026-02-20T21:28:50.409 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.62212.log.gz 2026-02-20T21:28:50.409 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.81760.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89490.log 2026-02-20T21:28:50.409 INFO:teuthology.orchestra.run.trial199.stderr: 80.0% -- replaced with /var/log/ceph/cluster2-client.mirror.81760.log.gz 2026-02-20T21:28:50.409 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.48088.log: gzip -5 --verbose 53.0% -- -- replaced with /var/log/ceph/cluster1-client.mirror.48088.log.gz /var/log/ceph/cluster1-client.mirror.88611.log 2026-02-20T21:28:50.409 INFO:teuthology.orchestra.run.trial199.stderr: 2026-02-20T21:28:50.409 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.89490.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71493.log 2026-02-20T21:28:50.410 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.88611.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.77764.log 2026-02-20T21:28:50.410 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89490.log.gz 2026-02-20T21:28:50.410 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.88611.log.gz 2026-02-20T21:28:50.410 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.71493.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64990.log 2026-02-20T21:28:50.410 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.71493.log.gz 2026-02-20T21:28:50.410 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.1.77764.log: 19.1% -- replaced with /var/log/ceph/cluster1-client.mirror.1.77764.log.gz 2026-02-20T21:28:50.410 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65524.log 2026-02-20T21:28:50.410 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.64990.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58935.log 2026-02-20T21:28:50.411 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.64990.log.gz 2026-02-20T21:28:50.411 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.65524.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64194.log 2026-02-20T21:28:50.411 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.65524.log.gz 2026-02-20T21:28:50.411 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.58935.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61700.log 2026-02-20T21:28:50.411 INFO:teuthology.orchestra.run.trial199.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.mirror.89066.log.gz 2026-02-20T21:28:50.411 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.58935.log.gz 2026-02-20T21:28:50.411 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.64194.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.81137.log 2026-02-20T21:28:50.411 INFO:teuthology.orchestra.run.trial199.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.64194.log.gz 2026-02-20T21:28:50.411 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.61700.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83352.log 2026-02-20T21:28:50.412 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.61700.log.gz 2026-02-20T21:28:50.412 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.1.81137.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74366.log 2026-02-20T21:28:50.412 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.83352.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69508.log 2026-02-20T21:28:50.412 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.83352.log.gz 2026-02-20T21:28:50.412 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.74366.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84234.log 2026-02-20T21:28:50.412 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.69508.log: 90.6% -- replaced with /var/log/ceph/cluster1-client.mirror.74366.log.gz 2026-02-20T21:28:50.412 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68568.log 2026-02-20T21:28:50.413 INFO:teuthology.orchestra.run.trial199.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.69508.log.gz 2026-02-20T21:28:50.413 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.84234.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65418.log 2026-02-20T21:28:50.413 INFO:teuthology.orchestra.run.trial199.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.84234.log.gz 2026-02-20T21:28:50.413 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.68568.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71364.log 2026-02-20T21:28:50.413 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.68568.log.gz 2026-02-20T21:28:50.413 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.65418.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67104.log 2026-02-20T21:28:50.413 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.65418.log.gz 2026-02-20T21:28:50.414 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.71364.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57600.log 2026-02-20T21:28:50.414 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.71364.log.gz 2026-02-20T21:28:50.414 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.67104.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51280.log 2026-02-20T21:28:50.414 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.57600.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54830.log 2026-02-20T21:28:50.414 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57600.log.gz 2026-02-20T21:28:50.414 INFO:teuthology.orchestra.run.trial199.stderr: 89.6%/var/log/ceph/cluster2-client.admin.51280.log: -- replaced with /var/log/ceph/cluster2-client.mirror.67104.log.gz 2026-02-20T21:28:50.414 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75489.log 2026-02-20T21:28:50.414 INFO:teuthology.orchestra.run.trial199.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.51280.log.gz 2026-02-20T21:28:50.414 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.54830.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.79091.log 2026-02-20T21:28:50.415 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.54830.log.gz 2026-02-20T21:28:50.415 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.75489.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82272.log 2026-02-20T21:28:50.415 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.75489.log.gz 2026-02-20T21:28:50.415 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.79091.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73846.log 2026-02-20T21:28:50.415 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.79091.log.gz 2026-02-20T21:28:50.415 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.82272.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89240.log 2026-02-20T21:28:50.415 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.82272.log.gz 2026-02-20T21:28:50.415 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.73846.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73923.log 2026-02-20T21:28:50.416 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.89240.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.73846.log.gz 2026-02-20T21:28:50.416 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75653.log 2026-02-20T21:28:50.416 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.73923.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48361.log 2026-02-20T21:28:50.416 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89240.log.gz 2026-02-20T21:28:50.416 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75653.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59371.log 2026-02-20T21:28:50.416 INFO:teuthology.orchestra.run.trial199.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.73923.log.gz 2026-02-20T21:28:50.416 INFO:teuthology.orchestra.run.trial199.stderr: 94.6% -- replaced with /var/log/ceph/cluster2-client.mirror.3.54295.log.gz 94.1% 2026-02-20T21:28:50.416 INFO:teuthology.orchestra.run.trial199.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.3.50864.log.gz 2026-02-20T21:28:50.417 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.48361.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48361.log.gz 2026-02-20T21:28:50.417 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75386.log 2026-02-20T21:28:50.417 INFO:teuthology.orchestra.run.trial199.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.75653.log.gz 2026-02-20T21:28:50.417 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.59371.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69150.log 2026-02-20T21:28:50.417 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.59371.log.gz 2026-02-20T21:28:50.417 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75386.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72169.log 2026-02-20T21:28:50.417 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.69150.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63289.log 2026-02-20T21:28:50.417 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.69150.log.gz 2026-02-20T21:28:50.417 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.72169.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75407.log 2026-02-20T21:28:50.418 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.72169.log.gz 2026-02-20T21:28:50.418 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.63289.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59535.log 2026-02-20T21:28:50.418 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75407.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.63289.log.gz 2026-02-20T21:28:50.418 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66997.log 2026-02-20T21:28:50.418 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.59535.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73397.log 2026-02-20T21:28:50.418 INFO:teuthology.orchestra.run.trial199.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.75407.log.gz 2026-02-20T21:28:50.418 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.59535.log.gz 2026-02-20T21:28:50.418 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.66997.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55172.log 2026-02-20T21:28:50.419 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.66997.log.gz 2026-02-20T21:28:50.419 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.73397.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48933.log 2026-02-20T21:28:50.419 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.55172.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73498.log 2026-02-20T21:28:50.419 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.48933.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65313.log 2026-02-20T21:28:50.419 INFO:teuthology.orchestra.run.trial199.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.73397.log.gz 2026-02-20T21:28:50.419 INFO:teuthology.orchestra.run.trial199.stderr: 69.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48933.log.gz 2026-02-20T21:28:50.419 INFO:teuthology.orchestra.run.trial199.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.55172.log.gz 2026-02-20T21:28:50.419 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.73498.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48344.log 2026-02-20T21:28:50.420 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.65313.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73498.log.gz 2026-02-20T21:28:50.420 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74430.log 2026-02-20T21:28:50.420 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65313.log.gz 2026-02-20T21:28:50.420 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.48344.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63235.log 2026-02-20T21:28:50.420 INFO:teuthology.orchestra.run.trial199.stderr: 64.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48344.log.gz 2026-02-20T21:28:50.420 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.74430.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62974.log 2026-02-20T21:28:50.420 INFO:teuthology.orchestra.run.trial199.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74430.log.gz 2026-02-20T21:28:50.420 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.63235.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73963.log 2026-02-20T21:28:50.421 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.63235.log.gz 2026-02-20T21:28:50.421 INFO:teuthology.orchestra.run.trial199.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75386.log.gz 2026-02-20T21:28:50.421 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.62974.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76212.log 2026-02-20T21:28:50.421 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.73963.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64963.log 2026-02-20T21:28:50.421 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73963.log.gz 2026-02-20T21:28:50.421 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76212.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55722.log 2026-02-20T21:28:50.421 INFO:teuthology.orchestra.run.trial199.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.62974.log.gz 2026-02-20T21:28:50.421 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.64963.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74790.log 2026-02-20T21:28:50.422 INFO:teuthology.orchestra.run.trial199.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.76212.log.gz 2026-02-20T21:28:50.422 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.64963.log.gz 2026-02-20T21:28:50.422 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.55722.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79517.log 2026-02-20T21:28:50.422 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.55722.log.gz 2026-02-20T21:28:50.422 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.74790.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66891.log 2026-02-20T21:28:50.422 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.74790.log.gz 2026-02-20T21:28:50.422 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.79517.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46434.log 2026-02-20T21:28:50.423 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.66891.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47981.log 2026-02-20T21:28:50.423 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.66891.log.gz 2026-02-20T21:28:50.423 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.46434.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46434.log.gz 2026-02-20T21:28:50.423 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55575.log 2026-02-20T21:28:50.423 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.47981.log: 63.6% -- replaced with /var/log/ceph/cluster1-client.mirror.47981.log.gz 2026-02-20T21:28:50.423 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75161.log 2026-02-20T21:28:50.423 INFO:teuthology.orchestra.run.trial199.stderr: 93.9% -- replaced with /var/log/ceph/cluster1-client.mirror.2.50796.log.gz 2026-02-20T21:28:50.423 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.55575.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58215.log 2026-02-20T21:28:50.423 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.55575.log.gz 2026-02-20T21:28:50.424 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75161.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85437.log 2026-02-20T21:28:50.424 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.58215.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57576.log 2026-02-20T21:28:50.424 INFO:teuthology.orchestra.run.trial199.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.75161.log.gz 2026-02-20T21:28:50.424 INFO:teuthology.orchestra.run.trial199.stderr: 87.4%/var/log/ceph/cluster1-client.mirror.85437.log: -- replaced with /var/log/ceph/cluster2-client.admin.58215.log.gz 2026-02-20T21:28:50.424 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78954.log 2026-02-20T21:28:50.424 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.57576.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53653.log 2026-02-20T21:28:50.425 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.57576.log.gz 2026-02-20T21:28:50.425 INFO:teuthology.orchestra.run.trial199.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.89510.log.gz 2026-02-20T21:28:50.425 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.78954.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56049.log 2026-02-20T21:28:50.425 INFO:teuthology.orchestra.run.trial199.stderr: 91.2% -- replaced with /var/log/ceph/cluster1-client.mirror.85437.log.gz 2026-02-20T21:28:50.425 INFO:teuthology.orchestra.run.trial199.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.78954.log.gz 2026-02-20T21:28:50.425 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.53653.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56492.log 2026-02-20T21:28:50.425 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.56049.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64023.log 2026-02-20T21:28:50.425 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.56049.log.gz 2026-02-20T21:28:50.425 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.56492.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49098.log 2026-02-20T21:28:50.426 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.56492.log.gz 2026-02-20T21:28:50.426 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.64023.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79198.log 2026-02-20T21:28:50.426 INFO:teuthology.orchestra.run.trial199.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53653.log.gz 2026-02-20T21:28:50.426 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.49098.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82572.log 2026-02-20T21:28:50.426 INFO:teuthology.orchestra.run.trial199.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.79517.log.gz 2026-02-20T21:28:50.426 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.64023.log.gz 2026-02-20T21:28:50.426 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.79198.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60946.log 2026-02-20T21:28:50.426 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.82572.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88215.log 2026-02-20T21:28:50.427 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.60946.log: gzip 89.6% -5 --verbose -- replaced with /var/log/ceph/cluster1-client.mirror.79198.log.gz -- 2026-02-20T21:28:50.427 INFO:teuthology.orchestra.run.trial199.stderr: /var/log/ceph/cluster2-client.mirror.72844.log 2026-02-20T21:28:50.427 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.88215.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.60946.log.gz 2026-02-20T21:28:50.427 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.89713.log 2026-02-20T21:28:50.427 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.88215.log.gz 2026-02-20T21:28:50.427 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.72844.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.77531.log 2026-02-20T21:28:50.427 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.0.89713.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89531.log 2026-02-20T21:28:50.427 INFO:teuthology.orchestra.run.trial199.stderr: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.82572.log.gz 2026-02-20T21:28:50.428 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72844.log.gz 2026-02-20T21:28:50.428 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.77531.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51556.log 2026-02-20T21:28:50.428 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.89531.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52313.log 2026-02-20T21:28:50.428 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.77531.log.gz 2026-02-20T21:28:50.428 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.51556.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72807.log 2026-02-20T21:28:50.428 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.89531.log.gz 2026-02-20T21:28:50.428 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.52313.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57550.log 2026-02-20T21:28:50.429 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.72807.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61859.log 2026-02-20T21:28:50.429 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.52313.log.gz 2026-02-20T21:28:50.429 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.57550.log: gzip 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51556.log.gz -5 2026-02-20T21:28:50.429 INFO:teuthology.orchestra.run.trial199.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.69767.log 2026-02-20T21:28:50.429 INFO:teuthology.orchestra.run.trial199.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.72807.log.gz 2026-02-20T21:28:50.429 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.61859.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81906.log 2026-02-20T21:28:50.429 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.69767.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.90262.log 2026-02-20T21:28:50.429 INFO:teuthology.orchestra.run.trial199.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.61859.log.gz 2026-02-20T21:28:50.429 INFO:teuthology.orchestra.run.trial199.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.0.89713.log.gz 2026-02-20T21:28:50.429 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.81906.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52748.log 2026-02-20T21:28:50.429 INFO:teuthology.orchestra.run.trial199.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.57550.log.gz 2026-02-20T21:28:50.430 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.90262.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74814.log 2026-02-20T21:28:50.430 INFO:teuthology.orchestra.run.trial199.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.69767.log.gz 2026-02-20T21:28:50.430 INFO:teuthology.orchestra.run.trial199.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.49098.log.gz 2026-02-20T21:28:50.430 INFO:teuthology.orchestra.run.trial199.stderr: 87.7%/var/log/ceph/cluster1-client.admin.52748.log: -- replaced with /var/log/ceph/cluster1-client.admin.90262.log.gz 2026-02-20T21:28:50.430 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86921.log 2026-02-20T21:28:50.430 INFO:teuthology.orchestra.run.trial199.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.52748.log.gz 2026-02-20T21:28:50.430 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.74814.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82420.log 2026-02-20T21:28:50.430 INFO:teuthology.orchestra.run.trial199.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.74814.log.gz 2026-02-20T21:28:50.430 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51253.log 2026-02-20T21:28:50.431 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.86921.log: /var/log/ceph/cluster2-client.mirror.82420.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73108.log 2026-02-20T21:28:50.431 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.86921.log.gz 2026-02-20T21:28:50.431 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.51253.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82919.log 2026-02-20T21:28:50.431 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.73108.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.51253.log.gz 2026-02-20T21:28:50.431 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52007.log 2026-02-20T21:28:50.431 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.82919.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76512.log 2026-02-20T21:28:50.432 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.82919.log.gz 2026-02-20T21:28:50.432 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.52007.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64248.log 2026-02-20T21:28:50.432 INFO:teuthology.orchestra.run.trial199.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.73108.log.gz 2026-02-20T21:28:50.432 INFO:teuthology.orchestra.run.trial199.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.82420.log.gz 2026-02-20T21:28:50.432 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.76512.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.52007.log.gz 2026-02-20T21:28:50.432 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.54161.log 2026-02-20T21:28:50.432 INFO:teuthology.orchestra.run.trial199.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.76512.log.gz 2026-02-20T21:28:50.432 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.64248.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54296.log 2026-02-20T21:28:50.432 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.0.54161.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.64248.log.gz 2026-02-20T21:28:50.432 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73574.log 2026-02-20T21:28:50.432 INFO:teuthology.orchestra.run.trial199.stderr: 18.6% -- replaced with /var/log/ceph/cluster2-client.mirror.0.54161.log.gz 2026-02-20T21:28:50.432 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.54296.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76936.log 2026-02-20T21:28:50.433 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.73574.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71679.log 2026-02-20T21:28:50.433 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76936.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84535.log 2026-02-20T21:28:50.433 INFO:teuthology.orchestra.run.trial199.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.73574.log.gz 2026-02-20T21:28:50.433 INFO:teuthology.orchestra.run.trial199.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.54296.log.gz 2026-02-20T21:28:50.433 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.71679.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76936.log.gz 2026-02-20T21:28:50.433 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65364.log 2026-02-20T21:28:50.433 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.84535.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68323.log 2026-02-20T21:28:50.433 INFO:teuthology.orchestra.run.trial199.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.81906.log.gz 2026-02-20T21:28:50.433 INFO:teuthology.orchestra.run.trial199.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71679.log.gz 2026-02-20T21:28:50.433 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.84535.log.gz 2026-02-20T21:28:50.433 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.65364.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72217.log 2026-02-20T21:28:50.434 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.68323.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.65364.log.gz 2026-02-20T21:28:50.434 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88742.log 2026-02-20T21:28:50.434 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.72217.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76422.log 2026-02-20T21:28:50.434 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.72217.log.gz 2026-02-20T21:28:50.434 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.88742.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83979.log 2026-02-20T21:28:50.434 INFO:teuthology.orchestra.run.trial199.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.68323.log.gz 2026-02-20T21:28:50.434 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.76422.log: gzip 87.3% -5 -- replaced with /var/log/ceph/cluster2-client.admin.88742.log.gz --verbose 2026-02-20T21:28:50.434 INFO:teuthology.orchestra.run.trial199.stderr: -- /var/log/ceph/cluster1-client.admin.51827.log 2026-02-20T21:28:50.434 INFO:teuthology.orchestra.run.trial199.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.76422.log.gz 2026-02-20T21:28:50.435 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.83979.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67455.log 2026-02-20T21:28:50.435 INFO:teuthology.orchestra.run.trial199.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.83979.log.gz 2026-02-20T21:28:50.435 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.51827.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51800.log 2026-02-20T21:28:50.435 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.67455.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61940.log 2026-02-20T21:28:50.435 INFO:teuthology.orchestra.run.trial199.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.51827.log.gz 2026-02-20T21:28:50.435 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.67455.log.gz 2026-02-20T21:28:50.435 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.51800.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64818.log 2026-02-20T21:28:50.435 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.61940.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.51800.log.gz 2026-02-20T21:28:50.435 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62011.log 2026-02-20T21:28:50.436 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.64818.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87979.log 2026-02-20T21:28:50.436 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.61940.log.gz 2026-02-20T21:28:50.436 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.62011.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48015.log 2026-02-20T21:28:50.436 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.62011.log.gz 2026-02-20T21:28:50.436 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60732.log 2026-02-20T21:28:50.436 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.87979.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64818.log.gz 2026-02-20T21:28:50.436 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.48015.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85671.log 2026-02-20T21:28:50.436 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.87979.log.gz 0.0% 2026-02-20T21:28:50.437 INFO:teuthology.orchestra.run.trial199.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.48015.log.gz 2026-02-20T21:28:50.437 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.60732.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78453.log 2026-02-20T21:28:50.437 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.85671.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55269.log 2026-02-20T21:28:50.437 INFO:teuthology.orchestra.run.trial199.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.60732.log.gz 2026-02-20T21:28:50.437 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.78453.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.85671.log.gz 2026-02-20T21:28:50.437 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46529.log 2026-02-20T21:28:50.437 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83952.log 2026-02-20T21:28:50.437 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.55269.log: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.78453.log.gz 2026-02-20T21:28:50.438 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.46529.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64361.log 2026-02-20T21:28:50.438 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46529.log.gz 2026-02-20T21:28:50.438 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.55269.log.gz 2026-02-20T21:28:50.438 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.83952.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82684.log 2026-02-20T21:28:50.438 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.83952.log.gz 2026-02-20T21:28:50.438 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.64361.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75468.log 2026-02-20T21:28:50.438 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.82684.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73290.log 2026-02-20T21:28:50.438 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.64361.log.gz 2026-02-20T21:28:50.438 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75468.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49708.log 2026-02-20T21:28:50.438 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82684.log.gz 2026-02-20T21:28:50.438 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82167.log 2026-02-20T21:28:50.439 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.73290.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75468.log.gz 2026-02-20T21:28:50.439 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.49708.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62474.log 2026-02-20T21:28:50.439 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.49708.log.gz 2026-02-20T21:28:50.439 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.82167.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83400.log 2026-02-20T21:28:50.439 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.82167.log.gz 2026-02-20T21:28:50.439 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.62474.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78160.log 2026-02-20T21:28:50.439 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.83400.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.62474.log.gz 2026-02-20T21:28:50.439 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50620.log 2026-02-20T21:28:50.440 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.83400.log.gz 2026-02-20T21:28:50.440 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.78160.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76445.log 2026-02-20T21:28:50.440 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.78160.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.73290.log.gz 2026-02-20T21:28:50.440 INFO:teuthology.orchestra.run.trial199.stderr: 2026-02-20T21:28:50.440 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.50620.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68255.log 2026-02-20T21:28:50.440 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.76445.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.90901.log 2026-02-20T21:28:50.440 INFO:teuthology.orchestra.run.trial199.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.76445.log.gz 2026-02-20T21:28:50.440 INFO:teuthology.orchestra.run.trial199.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50620.log.gz 2026-02-20T21:28:50.440 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.68255.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49792.log 2026-02-20T21:28:50.441 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.90901.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75222.log 2026-02-20T21:28:50.441 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.90901.log.gz 2026-02-20T21:28:50.441 INFO:teuthology.orchestra.run.trial199.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.68255.log.gz 2026-02-20T21:28:50.441 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.49792.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62687.log 2026-02-20T21:28:50.441 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75222.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71466.log 2026-02-20T21:28:50.441 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.62687.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54104.log 2026-02-20T21:28:50.441 INFO:teuthology.orchestra.run.trial199.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75222.log.gz 2026-02-20T21:28:50.441 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.62687.log.gz 2026-02-20T21:28:50.441 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.71466.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50063.log 2026-02-20T21:28:50.441 INFO:teuthology.orchestra.run.trial199.stderr: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.49792.log.gz 2026-02-20T21:28:50.441 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.54104.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.71466.log.gz 2026-02-20T21:28:50.442 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82046.log 2026-02-20T21:28:50.442 INFO:teuthology.orchestra.run.trial199.stderr: 87.4%/var/log/ceph/cluster2-client.mirror.50063.log: -- replaced with /var/log/ceph/cluster1-client.admin.54104.log.gz 2026-02-20T21:28:50.442 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71749.log 2026-02-20T21:28:50.442 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50063.log.gz 2026-02-20T21:28:50.442 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.82046.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76747.log 2026-02-20T21:28:50.442 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.71749.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68987.log 2026-02-20T21:28:50.442 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.82046.log.gz 2026-02-20T21:28:50.442 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.71749.log.gz 2026-02-20T21:28:50.442 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76747.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62186.log 2026-02-20T21:28:50.443 INFO:teuthology.orchestra.run.trial199.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76747.log.gz 2026-02-20T21:28:50.443 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.68987.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49589.log 2026-02-20T21:28:50.443 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.68987.log.gz 2026-02-20T21:28:50.443 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.62186.log: gzip -5 --verbose -- /var/log/ceph/cluster2-osd.0.log 2026-02-20T21:28:50.443 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.49589.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90363.log 2026-02-20T21:28:50.443 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72489.log 2026-02-20T21:28:50.443 INFO:teuthology.orchestra.run.trial199.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.62186.log.gz 2026-02-20T21:28:50.443 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57935.log 2026-02-20T21:28:50.444 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.90363.log: /var/log/ceph/cluster1-client.mirror.72489.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85417.log 2026-02-20T21:28:50.444 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.57935.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.91441.log 2026-02-20T21:28:50.444 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76294.log 2026-02-20T21:28:50.444 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.85417.log: /var/log/ceph/cluster2-client.admin.91441.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57935.log.gz 2026-02-20T21:28:50.445 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.91441.log.gz 2026-02-20T21:28:50.445 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76559.log 2026-02-20T21:28:50.445 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52920.log/var/log/ceph/cluster2-client.mirror.76294.log: 2026-02-20T21:28:50.445 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76559.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.80044.log 2026-02-20T21:28:50.445 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.52920.log: gzip -5 --verbose -- 89.5% /var/log/ceph/cluster1-client.mirror.83067.log 2026-02-20T21:28:50.445 INFO:teuthology.orchestra.run.trial199.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.76294.log.gz 2026-02-20T21:28:50.445 INFO:teuthology.orchestra.run.trial199.stderr: 91.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85417.log.gz 2026-02-20T21:28:50.445 INFO:teuthology.orchestra.run.trial199.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.76559.log.gz 2026-02-20T21:28:50.445 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.52920.log.gz/var/log/ceph/cluster1-client.mirror.1.80044.log: 2026-02-20T21:28:50.446 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77456.log 2026-02-20T21:28:50.446 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.83067.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69123.log 2026-02-20T21:28:50.446 INFO:teuthology.orchestra.run.trial199.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.83067.log.gz 2026-02-20T21:28:50.446 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.77456.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82354.log 2026-02-20T21:28:50.446 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.77456.log.gz 2026-02-20T21:28:50.446 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.69123.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43171.log 2026-02-20T21:28:50.446 INFO:teuthology.orchestra.run.trial199.stderr: 88.5% 91.6% -- replaced with /var/log/ceph/cluster1-client.admin.69123.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.1.81137.log.gz 2026-02-20T21:28:50.446 INFO:teuthology.orchestra.run.trial199.stderr: 2026-02-20T21:28:50.447 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.82354.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52866.log 2026-02-20T21:28:50.447 INFO:teuthology.orchestra.run.trial199.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.49589.log.gz 2026-02-20T21:28:50.447 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.43171.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43171.log.gz 2026-02-20T21:28:50.447 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59559.log 2026-02-20T21:28:50.447 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.52866.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88405.log 2026-02-20T21:28:50.447 INFO:teuthology.orchestra.run.trial199.stderr: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.82354.log.gz 2026-02-20T21:28:50.447 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.59559.log: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.90363.log.gz 2026-02-20T21:28:50.447 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.52866.log.gz 2026-02-20T21:28:50.447 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.50734.log 2026-02-20T21:28:50.448 INFO:teuthology.orchestra.run.trial199.stderr: 94.3% -- replaced with /var/log/ceph/cluster1-client.mirror.72489.log.gz 2026-02-20T21:28:50.448 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.88405.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.59559.log.gz 2026-02-20T21:28:50.448 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62447.log 2026-02-20T21:28:50.448 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.88405.log.gz 2026-02-20T21:28:50.448 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.1.50734.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.86080.log 2026-02-20T21:28:50.448 INFO:teuthology.orchestra.run.trial199.stderr: 91.3% -- replaced with /var/log/ceph/cluster1-client.mirror.1.80044.log.gz 2026-02-20T21:28:50.448 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.62447.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64658.log 2026-02-20T21:28:50.448 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.62447.log.gz 2026-02-20T21:28:50.448 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.0.86080.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52466.log 2026-02-20T21:28:50.449 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.64658.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78273.log 2026-02-20T21:28:50.449 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.52466.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73785.log 2026-02-20T21:28:50.449 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.64658.log.gz 2026-02-20T21:28:50.449 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.52466.log.gz 2026-02-20T21:28:50.449 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.78273.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55119.log 2026-02-20T21:28:50.449 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78273.log.gz 2026-02-20T21:28:50.449 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.73785.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75575.log 2026-02-20T21:28:50.449 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.55119.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69434.log 2026-02-20T21:28:50.450 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73785.log.gz 2026-02-20T21:28:50.450 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75575.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.55119.log.gz 2026-02-20T21:28:50.450 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54131.log 2026-02-20T21:28:50.450 INFO:teuthology.orchestra.run.trial199.stderr: 80.5% -- replaced with /var/log/ceph/cluster2-client.mirror.75575.log.gz 2026-02-20T21:28:50.450 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.69434.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74742.log 2026-02-20T21:28:50.450 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.54131.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86526.log 2026-02-20T21:28:50.450 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.74742.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.54131.log.gz 2026-02-20T21:28:50.450 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53521.log 2026-02-20T21:28:50.451 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.74742.log.gz 2026-02-20T21:28:50.451 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.86526.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53678.log 2026-02-20T21:28:50.451 INFO:teuthology.orchestra.run.trial199.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.69434.log.gz 2026-02-20T21:28:50.451 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.86526.log.gz 2026-02-20T21:28:50.451 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49921.log 2026-02-20T21:28:50.451 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.53521.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53521.log.gz 2026-02-20T21:28:50.451 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.53678.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61835.log 2026-02-20T21:28:50.451 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.53678.log.gz 2026-02-20T21:28:50.451 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.49921.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70882.log 2026-02-20T21:28:50.451 INFO:teuthology.orchestra.run.trial199.stderr: 54.4% -- replaced with /var/log/ceph/cluster1-client.mirror.49921.log.gz 2026-02-20T21:28:50.452 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.61835.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51674.log 2026-02-20T21:28:50.452 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.61835.log.gz 2026-02-20T21:28:50.452 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.70882.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57345.log 2026-02-20T21:28:50.452 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.51674.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.70882.log.gzgzip 2026-02-20T21:28:50.452 INFO:teuthology.orchestra.run.trial199.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50449.log 2026-02-20T21:28:50.452 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.57345.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60025.log 2026-02-20T21:28:50.452 INFO:teuthology.orchestra.run.trial199.stderr: 90.6% -- replaced with /var/log/ceph/cluster1-client.admin.51674.log.gz 2026-02-20T21:28:50.452 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.50449.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69013.log 2026-02-20T21:28:50.452 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.60025.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.57345.log.gz 2026-02-20T21:28:50.453 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70017.log 2026-02-20T21:28:50.453 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.60025.log.gz 2026-02-20T21:28:50.453 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.69013.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47558.log 2026-02-20T21:28:50.453 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.70017.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57985.log 2026-02-20T21:28:50.453 INFO:teuthology.orchestra.run.trial199.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.50449.log.gz 2026-02-20T21:28:50.453 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.47558.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64765.log 2026-02-20T21:28:50.453 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.69013.log.gz 2026-02-20T21:28:50.453 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47558.log.gz 2026-02-20T21:28:50.454 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.57985.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.70017.log.gz 2026-02-20T21:28:50.454 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56177.log 2026-02-20T21:28:50.454 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.57985.log.gz 2026-02-20T21:28:50.454 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.64765.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83162.log 2026-02-20T21:28:50.454 INFO:teuthology.orchestra.run.trial199.stderr: 94.4% -- replaced with /var/log/ceph/cluster1-client.mirror.0.88010.log.gz 87.5% 2026-02-20T21:28:50.454 INFO:teuthology.orchestra.run.trial199.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.64765.log.gz 2026-02-20T21:28:50.454 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64912.log 2026-02-20T21:28:50.454 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.56177.log: /var/log/ceph/cluster2-client.mirror.83162.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79007.log 2026-02-20T21:28:50.454 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.56177.log.gz 2026-02-20T21:28:50.454 INFO:teuthology.orchestra.run.trial199.stderr: 65.3% -- replaced with /var/log/ceph/cluster2-client.mirror.83162.log.gz 2026-02-20T21:28:50.454 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.64912.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82704.log 2026-02-20T21:28:50.455 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.64912.log.gz 2026-02-20T21:28:50.455 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.79007.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71249.log 2026-02-20T21:28:50.455 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.82704.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76579.log 2026-02-20T21:28:50.455 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.82704.log.gz 2026-02-20T21:28:50.455 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.71249.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67599.log 2026-02-20T21:28:50.455 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.71249.log.gz 2026-02-20T21:28:50.455 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76579.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69910.log 2026-02-20T21:28:50.455 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66864.log 2026-02-20T21:28:50.455 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.67599.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.76579.log.gz 2026-02-20T21:28:50.456 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.69910.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.67599.log.gz 2026-02-20T21:28:50.456 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66432.log 2026-02-20T21:28:50.456 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.66864.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.69910.log.gz 2026-02-20T21:28:50.456 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75366.log 2026-02-20T21:28:50.456 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.66864.log.gz 2026-02-20T21:28:50.456 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.66432.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70070.log 2026-02-20T21:28:50.456 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.66432.log.gz 2026-02-20T21:28:50.456 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75366.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75968.log 2026-02-20T21:28:50.457 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.75366.log.gz 2026-02-20T21:28:50.457 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.70070.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68933.log 2026-02-20T21:28:50.457 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75968.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.70070.log.gz 2026-02-20T21:28:50.457 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63627.log 2026-02-20T21:28:50.457 INFO:teuthology.orchestra.run.trial199.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75968.log.gz 2026-02-20T21:28:50.457 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58837.log 2026-02-20T21:28:50.457 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.68933.log: /var/log/ceph/cluster2-client.admin.63627.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88988.log 2026-02-20T21:28:50.457 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.68933.log.gz 2026-02-20T21:28:50.457 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.58837.log: 87.6%gzip -- replaced with /var/log/ceph/cluster2-client.admin.63627.log.gz 2026-02-20T21:28:50.457 INFO:teuthology.orchestra.run.trial199.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85885.log 2026-02-20T21:28:50.457 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.58837.log.gz 2026-02-20T21:28:50.458 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.88988.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82599.log 2026-02-20T21:28:50.458 INFO:teuthology.orchestra.run.trial199.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.88988.log.gz 2026-02-20T21:28:50.458 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.85885.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85584.log 2026-02-20T21:28:50.458 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.82599.log: gzip 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.85885.log.gz -5 2026-02-20T21:28:50.458 INFO:teuthology.orchestra.run.trial199.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.73680.log 2026-02-20T21:28:50.458 INFO:teuthology.orchestra.run.trial199.stderr: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.82599.log.gz 2026-02-20T21:28:50.458 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.85584.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59971.log 2026-02-20T21:28:50.458 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.85584.log.gz 2026-02-20T21:28:50.458 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.73680.log: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.79007.log.gz 2026-02-20T21:28:50.458 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70310.log 2026-02-20T21:28:50.459 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.59971.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64221.log 2026-02-20T21:28:50.459 INFO:teuthology.orchestra.run.trial199.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.73680.log.gz 2026-02-20T21:28:50.459 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.59971.log.gz/var/log/ceph/cluster2-client.admin.70310.log: 2026-02-20T21:28:50.459 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47460.log 2026-02-20T21:28:50.459 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.70310.log.gz 2026-02-20T21:28:50.459 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.64221.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61212.log 2026-02-20T21:28:50.459 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.64221.log.gz/var/log/ceph/cluster2-client.admin.47460.log: 2026-02-20T21:28:50.459 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66674.log 2026-02-20T21:28:50.459 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47460.log.gz 2026-02-20T21:28:50.459 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.61212.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74323.log 2026-02-20T21:28:50.460 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.66674.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82119.log 2026-02-20T21:28:50.460 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.74323.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.66674.log.gz 2026-02-20T21:28:50.460 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82871.log 2026-02-20T21:28:50.460 INFO:teuthology.orchestra.run.trial199.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.61212.log.gz 2026-02-20T21:28:50.460 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.82119.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76852.log 2026-02-20T21:28:50.460 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.82119.log.gz 2026-02-20T21:28:50.460 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.74323.log.gz 2026-02-20T21:28:50.461 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55944.log 2026-02-20T21:28:50.461 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.82871.log: /var/log/ceph/cluster2-client.mirror.76852.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75741.log 2026-02-20T21:28:50.461 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.82871.log.gz 2026-02-20T21:28:50.461 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.55944.log: gzip 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76852.log.gz -5 2026-02-20T21:28:50.461 INFO:teuthology.orchestra.run.trial199.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.50218.log 2026-02-20T21:28:50.461 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.55944.log.gz 2026-02-20T21:28:50.461 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75741.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53342.log 2026-02-20T21:28:50.461 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.50218.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47947.log 2026-02-20T21:28:50.461 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50218.log.gz 2026-02-20T21:28:50.461 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.53342.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57052.log 2026-02-20T21:28:50.462 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.53342.log.gz 2026-02-20T21:28:50.462 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.47947.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68742.log 2026-02-20T21:28:50.462 INFO:teuthology.orchestra.run.trial199.stderr: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.47947.log.gz 2026-02-20T21:28:50.462 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.57052.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74533.log 2026-02-20T21:28:50.462 INFO:teuthology.orchestra.run.trial199.stderr: 87.7%/var/log/ceph/cluster1-client.mirror.68742.log: -- replaced with /var/log/ceph/cluster1-client.admin.57052.log.gz 2026-02-20T21:28:50.462 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59920.log 2026-02-20T21:28:50.462 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.68742.log.gz 2026-02-20T21:28:50.462 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.74533.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66192.log 2026-02-20T21:28:50.462 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.74533.log.gz 2026-02-20T21:28:50.462 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.59920.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70935.log 2026-02-20T21:28:50.463 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59920.log.gz 2026-02-20T21:28:50.463 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.66192.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78683.log 2026-02-20T21:28:50.463 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.70935.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.66192.log.gz 2026-02-20T21:28:50.463 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55296.log 2026-02-20T21:28:50.463 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.78683.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61491.log 2026-02-20T21:28:50.463 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.78683.log.gz 2026-02-20T21:28:50.463 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.55296.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70935.log.gz 2026-02-20T21:28:50.463 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.88769.log 2026-02-20T21:28:50.463 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.55296.log.gz 2026-02-20T21:28:50.464 INFO:teuthology.orchestra.run.trial199.stderr:gzip/var/log/ceph/cluster1-client.admin.61491.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81777.log 2026-02-20T21:28:50.464 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.88769.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47313.log 2026-02-20T21:28:50.464 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.61491.log.gz 2026-02-20T21:28:50.464 INFO:teuthology.orchestra.run.trial199.stderr: 87.6%/var/log/ceph/cluster2-client.mirror.81777.log: -- replaced with /var/log/ceph/cluster1-client.admin.88769.log.gz 2026-02-20T21:28:50.464 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77432.log 2026-02-20T21:28:50.464 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.47313.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73616.log 2026-02-20T21:28:50.464 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47313.log.gz 2026-02-20T21:28:50.464 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.77432.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81777.log.gz 2026-02-20T21:28:50.464 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76273.log 2026-02-20T21:28:50.464 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.77432.log.gz 2026-02-20T21:28:50.464 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.73616.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72145.log 2026-02-20T21:28:50.465 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76273.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67329.log 2026-02-20T21:28:50.465 INFO:teuthology.orchestra.run.trial199.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.75741.log.gz 2026-02-20T21:28:50.465 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.72145.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.89443.log 2026-02-20T21:28:50.465 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.72145.log.gz 2026-02-20T21:28:50.465 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.67329.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73353.log 2026-02-20T21:28:50.465 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.89443.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81797.log 2026-02-20T21:28:50.466 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.73353.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.89443.log.gz 2026-02-20T21:28:50.466 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64551.log 2026-02-20T21:28:50.466 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.67329.log.gz 2026-02-20T21:28:50.466 INFO:teuthology.orchestra.run.trial199.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.73616.log.gz 2026-02-20T21:28:50.466 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.81797.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52585.log 2026-02-20T21:28:50.466 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.81797.log.gz 2026-02-20T21:28:50.466 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.64551.log: gzip -5 --verbose 90.4% -- /var/log/ceph/cluster1-client.admin.80688.log -- replaced with /var/log/ceph/cluster2-client.mirror.76273.log.gz 2026-02-20T21:28:50.466 INFO:teuthology.orchestra.run.trial199.stderr: 2026-02-20T21:28:50.466 INFO:teuthology.orchestra.run.trial199.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.64551.log.gz 2026-02-20T21:28:50.466 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.52585.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61811.log 2026-02-20T21:28:50.467 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.80688.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56946.log 2026-02-20T21:28:50.467 INFO:teuthology.orchestra.run.trial199.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.52585.log.gz 2026-02-20T21:28:50.467 INFO:teuthology.orchestra.run.trial199.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.73353.log.gz 2026-02-20T21:28:50.467 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.61811.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65289.log 2026-02-20T21:28:50.467 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.80688.log.gz 2026-02-20T21:28:50.467 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.56946.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61811.log.gz 2026-02-20T21:28:50.467 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46725.log 2026-02-20T21:28:50.467 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.65289.log: 87.1% -- replaced with /var/log/ceph/cluster2-client.admin.56946.log.gz 2026-02-20T21:28:50.467 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74630.log 2026-02-20T21:28:50.467 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.65289.log.gz 2026-02-20T21:28:50.467 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.46725.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61545.log 2026-02-20T21:28:50.467 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46725.log.gz 2026-02-20T21:28:50.467 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.74630.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84633.log 2026-02-20T21:28:50.468 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.61545.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56972.log 2026-02-20T21:28:50.468 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 87.5% --verbose -- -- replaced with /var/log/ceph/cluster2-client.admin.61545.log.gz /var/log/ceph/cluster2-client.admin.90999.log 2026-02-20T21:28:50.468 INFO:teuthology.orchestra.run.trial199.stderr: 2026-02-20T21:28:50.468 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.84633.log: /var/log/ceph/cluster1-client.mirror.56972.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62527.log 2026-02-20T21:28:50.468 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.90999.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67996.log 2026-02-20T21:28:50.468 INFO:teuthology.orchestra.run.trial199.stderr: 27.2% -- replaced with /var/log/ceph/cluster2-client.admin.90999.log.gz 2026-02-20T21:28:50.468 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.62527.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61646.log 2026-02-20T21:28:50.468 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.56972.log.gz 2026-02-20T21:28:50.469 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.67996.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76073.log 2026-02-20T21:28:50.469 INFO:teuthology.orchestra.run.trial199.stderr: 91.2% -- replaced with /var/log/ceph/cluster2-client.mirror.84633.log.gz 2026-02-20T21:28:50.469 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.61646.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.67996.log.gz 2026-02-20T21:28:50.469 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71081.log 2026-02-20T21:28:50.469 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.62527.log.gz 2026-02-20T21:28:50.469 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.61646.log.gz 2026-02-20T21:28:50.469 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.76073.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.80716.log 2026-02-20T21:28:50.469 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.76073.log.gz 2026-02-20T21:28:50.469 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.71081.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71276.log 2026-02-20T21:28:50.470 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.71081.log.gz 2026-02-20T21:28:50.470 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.80716.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.81227.log 2026-02-20T21:28:50.470 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.71276.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.80716.log.gz 2026-02-20T21:28:50.470 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.80406.log 2026-02-20T21:28:50.470 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.3.81227.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.admin.71276.log.gz 2026-02-20T21:28:50.470 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76490.log 2026-02-20T21:28:50.470 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.80406.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76029.log 2026-02-20T21:28:50.470 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.80406.log.gz 2026-02-20T21:28:50.470 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.76490.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50369.log 2026-02-20T21:28:50.470 INFO:teuthology.orchestra.run.trial199.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.76490.log.gz 2026-02-20T21:28:50.471 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76029.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75887.log 2026-02-20T21:28:50.471 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.50369.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.74630.log.gz 2026-02-20T21:28:50.471 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61388.log 2026-02-20T21:28:50.471 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.76029.log.gz 2026-02-20T21:28:50.471 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75887.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.50369.log.gz 2026-02-20T21:28:50.471 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70124.log 2026-02-20T21:28:50.471 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.75887.log.gz 2026-02-20T21:28:50.471 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.61388.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.80016.log 2026-02-20T21:28:50.472 INFO:teuthology.orchestra.run.trial199.stderr: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.1.54190.log.gz 2026-02-20T21:28:50.472 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.70124.log: gzip -5 --verbose -- /var/log/ceph/cluster2-osd.1.log 2026-02-20T21:28:50.472 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.70124.log.gz 2026-02-20T21:28:50.472 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.1.80016.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48799.log 2026-02-20T21:28:50.472 INFO:teuthology.orchestra.run.trial199.stderr: 19.7% -- replaced with /var/log/ceph/cluster1-client.mirror.1.80016.log.gz 2026-02-20T21:28:50.472 INFO:teuthology.orchestra.run.trial199.stderr: 90.6% -- replaced with /var/log/ceph/cluster1-client.mirror.61388.log.gz 2026-02-20T21:28:50.472 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70738.log 2026-02-20T21:28:50.472 INFO:teuthology.orchestra.run.trial199.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.3.81227.log.gz 2026-02-20T21:28:50.472 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.48799.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66587.log 2026-02-20T21:28:50.472 INFO:teuthology.orchestra.run.trial199.stderr: 55.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48799.log.gz 2026-02-20T21:28:50.473 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.70738.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77040.log 2026-02-20T21:28:50.473 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.70738.log.gz 2026-02-20T21:28:50.473 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.66587.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62878.log 2026-02-20T21:28:50.473 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.77040.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51360.log 2026-02-20T21:28:50.474 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.62878.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.66587.log.gz 2026-02-20T21:28:50.474 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57838.log 2026-02-20T21:28:50.474 INFO:teuthology.orchestra.run.trial199.stderr: 89.8% 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.77040.log.gz 2026-02-20T21:28:50.474 INFO:teuthology.orchestra.run.trial199.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.62878.log.gz 2026-02-20T21:28:50.474 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.51360.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66255.log 2026-02-20T21:28:50.474 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.57838.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68397.log 2026-02-20T21:28:50.475 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.57838.log.gz 2026-02-20T21:28:50.475 INFO:teuthology.orchestra.run.trial199.stderr: 92.1%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.51360.log.gz 2026-02-20T21:28:50.475 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.66255.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85864.log 2026-02-20T21:28:50.475 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.68397.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.50682.log 2026-02-20T21:28:50.475 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.66255.log.gz 2026-02-20T21:28:50.475 INFO:teuthology.orchestra.run.trial199.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.68397.log.gz 2026-02-20T21:28:50.475 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.85864.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47215.log 2026-02-20T21:28:50.475 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.85864.log.gz 2026-02-20T21:28:50.475 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.0.50682.log: 19.1% -- replaced with /var/log/ceph/cluster1-client.mirror.0.50682.log.gz 2026-02-20T21:28:50.475 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55146.log 2026-02-20T21:28:50.476 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.47215.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47215.log.gz 2026-02-20T21:28:50.476 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81086.log 2026-02-20T21:28:50.476 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.55146.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72784.log 2026-02-20T21:28:50.476 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.55146.log.gz 2026-02-20T21:28:50.476 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.81086.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70435.log 2026-02-20T21:28:50.476 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.81086.log.gz 2026-02-20T21:28:50.476 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.72784.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81622.log 2026-02-20T21:28:50.477 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.70435.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89290.log 2026-02-20T21:28:50.477 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.70435.log.gz 2026-02-20T21:28:50.477 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.81622.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.72784.log.gz 2026-02-20T21:28:50.477 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70660.log 2026-02-20T21:28:50.477 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.81622.log.gz 2026-02-20T21:28:50.477 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.89290.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60919.log 2026-02-20T21:28:50.477 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.89290.log.gz 2026-02-20T21:28:50.477 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.70660.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75309.log 2026-02-20T21:28:50.477 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.70660.log.gz 2026-02-20T21:28:50.478 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.60919.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.86071.log 2026-02-20T21:28:50.478 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.75309.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.60919.log.gz 2026-02-20T21:28:50.478 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61673.log 2026-02-20T21:28:50.478 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.75309.log.gz 2026-02-20T21:28:50.478 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.0.86071.log: 19.7% -- replaced with /var/log/ceph/cluster2-client.mirror.0.86071.log.gz 2026-02-20T21:28:50.478 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58410.log 2026-02-20T21:28:50.478 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.61673.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57164.log 2026-02-20T21:28:50.479 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.61673.log.gz 2026-02-20T21:28:50.479 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.58410.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48242.log 2026-02-20T21:28:50.479 INFO:teuthology.orchestra.run.trial199.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.58410.log.gz 2026-02-20T21:28:50.479 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.57164.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69049.log 2026-02-20T21:28:50.479 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.48242.log: 62.9% -- replaced with /var/log/ceph/cluster2-client.mirror.48242.log.gz 2026-02-20T21:28:50.479 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67864.log 2026-02-20T21:28:50.479 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.69049.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49989.log 2026-02-20T21:28:50.479 INFO:teuthology.orchestra.run.trial199.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57164.log.gz 2026-02-20T21:28:50.480 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.67864.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47411.log 2026-02-20T21:28:50.480 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.67864.log.gz 2026-02-20T21:28:50.480 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.49989.log: 63.6% -- replaced with /var/log/ceph/cluster1-client.mirror.49989.log.gz 2026-02-20T21:28:50.480 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49636.log 2026-02-20T21:28:50.480 INFO:teuthology.orchestra.run.trial199.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.69049.log.gz 2026-02-20T21:28:50.480 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.47411.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47411.log.gz 2026-02-20T21:28:50.480 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79618.log 2026-02-20T21:28:50.480 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.49636.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58268.log 2026-02-20T21:28:50.481 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49636.log.gz 2026-02-20T21:28:50.481 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.79618.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75783.log 2026-02-20T21:28:50.481 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.79618.log.gz 2026-02-20T21:28:50.481 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.58268.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81817.log 2026-02-20T21:28:50.481 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.75783.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61057.log 2026-02-20T21:28:50.481 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.75783.log.gz 2026-02-20T21:28:50.481 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.58268.log.gz 2026-02-20T21:28:50.482 INFO:teuthology.orchestra.run.trial199.stderr:gzip/var/log/ceph/cluster2-client.mirror.81817.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70177.log 2026-02-20T21:28:50.482 INFO:teuthology.orchestra.run.trial199.stderr: 80.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81817.log.gz 2026-02-20T21:28:50.482 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.61057.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79666.log 2026-02-20T21:28:50.482 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61057.log.gz 2026-02-20T21:28:50.482 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.70177.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68692.log 2026-02-20T21:28:50.482 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.79666.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48468.log 2026-02-20T21:28:50.482 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.79666.log.gz 2026-02-20T21:28:50.482 INFO:teuthology.orchestra.run.trial199.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.70177.log.gz 2026-02-20T21:28:50.483 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.68692.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.86124.log 2026-02-20T21:28:50.483 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.48468.log: 57.4% -- replaced with /var/log/ceph/cluster2-client.admin.48468.log.gz 2026-02-20T21:28:50.483 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72271.log 2026-02-20T21:28:50.483 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.1.86124.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.68692.log.gz 2026-02-20T21:28:50.483 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81416.log 2026-02-20T21:28:50.483 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.72271.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82143.log 2026-02-20T21:28:50.483 INFO:teuthology.orchestra.run.trial199.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.72271.log.gz 2026-02-20T21:28:50.484 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.81416.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68103.log 2026-02-20T21:28:50.484 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.81416.log.gz 2026-02-20T21:28:50.484 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.82143.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73206.log 2026-02-20T21:28:50.484 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.82143.log.gz 2026-02-20T21:28:50.484 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.68103.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69364.log 2026-02-20T21:28:50.484 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.73206.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86849.log 2026-02-20T21:28:50.484 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.68103.log.gz 2026-02-20T21:28:50.484 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.69364.log: 90.6% -- replaced with /var/log/ceph/cluster1-client.mirror.73206.log.gz 2026-02-20T21:28:50.485 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43077.log 2026-02-20T21:28:50.485 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.69364.log.gz 2026-02-20T21:28:50.485 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.86849.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.81138.log 2026-02-20T21:28:50.485 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.86849.log.gz 2026-02-20T21:28:50.485 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.43077.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43077.log.gz 2026-02-20T21:28:50.485 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76008.log 2026-02-20T21:28:50.485 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.2.81138.log: 19.1% -- replaced with /var/log/ceph/cluster1-client.mirror.2.81138.log.gz 2026-02-20T21:28:50.485 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76873.log 2026-02-20T21:28:50.485 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76008.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78431.log 2026-02-20T21:28:50.486 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76873.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72244.log 2026-02-20T21:28:50.486 INFO:teuthology.orchestra.run.trial199.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.76873.log.gz 2026-02-20T21:28:50.486 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.78431.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85788.log 2026-02-20T21:28:50.486 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.72244.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47820.log 2026-02-20T21:28:50.486 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.85788.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.72244.log.gz 2026-02-20T21:28:50.487 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53703.log 2026-02-20T21:28:50.487 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.85788.log.gz 2026-02-20T21:28:50.487 INFO:teuthology.orchestra.run.trial199.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.78431.log.gz 2026-02-20T21:28:50.487 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48105.log 2026-02-20T21:28:50.487 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.47820.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47820.log.gz 2026-02-20T21:28:50.487 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.53703.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.53703.log.gz 2026-02-20T21:28:50.487 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77250.log 2026-02-20T21:28:50.487 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.48105.log: gzip 62.4% -5 -- replaced with /var/log/ceph/cluster1-client.admin.48105.log.gz --verbose 2026-02-20T21:28:50.487 INFO:teuthology.orchestra.run.trial199.stderr: -- /var/log/ceph/cluster1-client.mirror.90385.log 2026-02-20T21:28:50.488 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.77250.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60589.log 2026-02-20T21:28:50.488 INFO:teuthology.orchestra.run.trial199.stderr: 80.5% -- replaced with /var/log/ceph/cluster2-client.mirror.77250.log.gz 2026-02-20T21:28:50.488 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.90385.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58597.log 2026-02-20T21:28:50.488 INFO:teuthology.orchestra.run.trial199.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.90385.log.gz 2026-02-20T21:28:50.488 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.80661.log 2026-02-20T21:28:50.488 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.60589.log: /var/log/ceph/cluster1-client.admin.58597.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68820.log 2026-02-20T21:28:50.488 INFO:teuthology.orchestra.run.trial199.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.1.86124.log.gz 2026-02-20T21:28:50.489 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.58597.log.gz 2026-02-20T21:28:50.489 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.80661.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58517.log 2026-02-20T21:28:50.489 INFO:teuthology.orchestra.run.trial199.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.80661.log.gz 2026-02-20T21:28:50.489 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60589.log.gz 2026-02-20T21:28:50.489 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.68820.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75534.log 2026-02-20T21:28:50.489 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.68820.log.gz 2026-02-20T21:28:50.489 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.58517.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60536.log 2026-02-20T21:28:50.489 INFO:teuthology.orchestra.run.trial199.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.76008.log.gz 2026-02-20T21:28:50.489 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.75534.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83111.log 2026-02-20T21:28:50.490 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.75534.log.gz 2026-02-20T21:28:50.490 INFO:teuthology.orchestra.run.trial199.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.58517.log.gz 2026-02-20T21:28:50.490 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.60536.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.48499.log 2026-02-20T21:28:50.490 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.60536.log.gz 2026-02-20T21:28:50.490 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.83111.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58090.log 2026-02-20T21:28:50.490 INFO:teuthology.orchestra.run.trial199.stderr: 57.7% -- replaced with /var/log/ceph/cluster1-client.mirror.83111.log.gz 2026-02-20T21:28:50.490 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.1.48499.log: gzip -5 19.1% --verbose -- -- replaced with /var/log/ceph/cluster1-client.mirror.1.48499.log.gz /var/log/ceph/cluster2-client.admin.63879.log 2026-02-20T21:28:50.490 INFO:teuthology.orchestra.run.trial199.stderr: 2026-02-20T21:28:50.490 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.58090.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79028.log 2026-02-20T21:28:50.490 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.58090.log.gz 2026-02-20T21:28:50.491 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.63879.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72870.log 2026-02-20T21:28:50.491 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.63879.log.gz 2026-02-20T21:28:50.491 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.79028.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73658.log 2026-02-20T21:28:50.491 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.72870.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64527.log 2026-02-20T21:28:50.491 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72870.log.gz 2026-02-20T21:28:50.491 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.73658.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.79028.log.gz 2026-02-20T21:28:50.491 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75762.log 2026-02-20T21:28:50.492 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.64527.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61785.log 2026-02-20T21:28:50.492 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.64527.log.gz 2026-02-20T21:28:50.492 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75762.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71388.log 2026-02-20T21:28:50.492 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.61785.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71109.log 2026-02-20T21:28:50.492 INFO:teuthology.orchestra.run.trial199.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75762.log.gz 2026-02-20T21:28:50.492 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.71388.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.90316.log 2026-02-20T21:28:50.493 INFO:teuthology.orchestra.run.trial199.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.73658.log.gz 2026-02-20T21:28:50.493 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.71388.log.gz 2026-02-20T21:28:50.493 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.71109.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71439.log 2026-02-20T21:28:50.493 INFO:teuthology.orchestra.run.trial199.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.61785.log.gz 2026-02-20T21:28:50.493 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.71109.log.gz 2026-02-20T21:28:50.493 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.90316.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74671.log 2026-02-20T21:28:50.493 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.90316.log.gz 2026-02-20T21:28:50.493 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.71439.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51647.log 2026-02-20T21:28:50.493 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.71439.log.gz 2026-02-20T21:28:50.493 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.74671.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63102.log 2026-02-20T21:28:50.493 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.51647.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74408.log 2026-02-20T21:28:50.494 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.51647.log.gz 2026-02-20T21:28:50.494 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.63102.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47362.log 2026-02-20T21:28:50.494 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.74408.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.63102.log.gz 2026-02-20T21:28:50.494 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.91342.log 2026-02-20T21:28:50.494 INFO:teuthology.orchestra.run.trial199.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.1.50734.log.gz 2026-02-20T21:28:50.494 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.74408.log.gz 2026-02-20T21:28:50.494 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.47362.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47362.log.gz 2026-02-20T21:28:50.494 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79741.log 2026-02-20T21:28:50.495 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.91342.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.91342.log.gz 2026-02-20T21:28:50.495 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73477.log 2026-02-20T21:28:50.495 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.79741.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67249.log 2026-02-20T21:28:50.495 INFO:teuthology.orchestra.run.trial199.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.79741.log.gz 2026-02-20T21:28:50.495 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.73477.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68873.log 2026-02-20T21:28:50.495 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.67249.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.73477.log.gz 2026-02-20T21:28:50.495 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76726.log 2026-02-20T21:28:50.495 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.67249.log.gz 2026-02-20T21:28:50.496 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.68873.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86675.log 2026-02-20T21:28:50.496 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76726.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74282.log 2026-02-20T21:28:50.496 INFO:teuthology.orchestra.run.trial199.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76726.log.gz 2026-02-20T21:28:50.496 INFO:teuthology.orchestra.run.trial199.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.68873.log.gz 2026-02-20T21:28:50.496 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.86675.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.86675.log.gz 2026-02-20T21:28:50.496 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54008.log 2026-02-20T21:28:50.496 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.74282.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62367.log 2026-02-20T21:28:50.497 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.54008.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58009.log 2026-02-20T21:28:50.497 INFO:teuthology.orchestra.run.trial199.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.74282.log.gz 2026-02-20T21:28:50.497 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54008.log.gz 2026-02-20T21:28:50.497 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.62367.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74007.log 2026-02-20T21:28:50.497 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.58009.log: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.74671.log.gz 2026-02-20T21:28:50.497 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61518.log 2026-02-20T21:28:50.497 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.62367.log.gz 2026-02-20T21:28:50.497 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.74007.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.58009.log.gz 2026-02-20T21:28:50.498 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67730.log 2026-02-20T21:28:50.498 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.74007.log.gz 2026-02-20T21:28:50.498 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.61518.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74492.log 2026-02-20T21:28:50.498 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.61518.log.gz 2026-02-20T21:28:50.498 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.67730.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61272.log 2026-02-20T21:28:50.498 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.74492.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60482.log 2026-02-20T21:28:50.498 INFO:teuthology.orchestra.run.trial199.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.67730.log.gz 94.4% 2026-02-20T21:28:50.498 INFO:teuthology.orchestra.run.trial199.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.0.86080.log.gz 2026-02-20T21:28:50.498 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.61272.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63373.log 2026-02-20T21:28:50.499 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.74492.log.gz 2026-02-20T21:28:50.499 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.61272.log.gz 2026-02-20T21:28:50.499 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.60482.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.50691.log 2026-02-20T21:28:50.499 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.60482.log.gz 2026-02-20T21:28:50.499 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84075.log 2026-02-20T21:28:50.499 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.63373.log: /var/log/ceph/cluster1-client.mirror.0.50691.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.63373.log.gz 2026-02-20T21:28:50.499 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73182.log 2026-02-20T21:28:50.499 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.84075.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74512.log 2026-02-20T21:28:50.500 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.73182.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83145.log 2026-02-20T21:28:50.500 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.74512.log: 90.7% -- replaced with /var/log/ceph/cluster1-client.mirror.73182.log.gz 2026-02-20T21:28:50.500 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60999.log 2026-02-20T21:28:50.500 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.74512.log.gz 2026-02-20T21:28:50.500 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.83145.log: 4.7%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.83145.log.gz 2026-02-20T21:28:50.500 INFO:teuthology.orchestra.run.trial199.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50471.log 2026-02-20T21:28:50.500 INFO:teuthology.orchestra.run.trial199.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.84075.log.gz 2026-02-20T21:28:50.501 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.60999.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76172.log 2026-02-20T21:28:50.501 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.50471.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84322.log 2026-02-20T21:28:50.501 INFO:teuthology.orchestra.run.trial199.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.60999.log.gz 2026-02-20T21:28:50.501 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76172.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.50471.log.gz 2026-02-20T21:28:50.501 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55628.log 2026-02-20T21:28:50.501 INFO:teuthology.orchestra.run.trial199.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76172.log.gz 2026-02-20T21:28:50.501 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.84322.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55870.log 2026-02-20T21:28:50.501 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.84322.log.gz 2026-02-20T21:28:50.501 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.55628.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72532.log 2026-02-20T21:28:50.502 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.55870.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76978.log 2026-02-20T21:28:50.502 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.72532.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57265.log 2026-02-20T21:28:50.502 INFO:teuthology.orchestra.run.trial199.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.72532.log.gz 2026-02-20T21:28:50.502 INFO:teuthology.orchestra.run.trial199.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.55628.log.gz 2026-02-20T21:28:50.502 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76978.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71917.log 2026-02-20T21:28:50.502 INFO:teuthology.orchestra.run.trial199.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76978.log.gz 2026-02-20T21:28:50.503 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.57265.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.50798.log 2026-02-20T21:28:50.503 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.71917.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.57265.log.gz 2026-02-20T21:28:50.503 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78297.log 2026-02-20T21:28:50.503 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.71917.log.gz 2026-02-20T21:28:50.503 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.3.50798.log: gzip 19.7% -5 -- replaced with /var/log/ceph/cluster1-client.mirror.3.50798.log.gz --verbose 2026-02-20T21:28:50.503 INFO:teuthology.orchestra.run.trial199.stderr: -- /var/log/ceph/cluster1-client.admin.61186.log 2026-02-20T21:28:50.503 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.78297.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49004.log 2026-02-20T21:28:50.503 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.61186.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.78297.log.gz 2026-02-20T21:28:50.504 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58386.log 2026-02-20T21:28:50.504 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.61186.log.gz 2026-02-20T21:28:50.504 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.49004.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75181.log 2026-02-20T21:28:50.504 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.58386.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49004.log.gz 2026-02-20T21:28:50.504 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.84831.log 2026-02-20T21:28:50.504 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58386.log.gz 2026-02-20T21:28:50.504 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75181.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56791.log 2026-02-20T21:28:50.504 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.0.84831.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51385.log 2026-02-20T21:28:50.505 INFO:teuthology.orchestra.run.trial199.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75181.log.gz 2026-02-20T21:28:50.505 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.56791.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61624.log 2026-02-20T21:28:50.505 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.51385.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.88055.log 2026-02-20T21:28:50.505 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.79118.log 2026-02-20T21:28:50.505 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.61624.log: 94.4% -- replaced with /var/log/ceph/cluster1-client.mirror.55870.log.gz 2026-02-20T21:28:50.505 INFO:teuthology.orchestra.run.trial199.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.56791.log.gz 2026-02-20T21:28:50.506 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.2.88055.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60282.log 2026-02-20T21:28:50.506 INFO:teuthology.orchestra.run.trial199.stderr: 19.7% -- replaced with /var/log/ceph/cluster1-client.mirror.2.88055.log.gz 2026-02-20T21:28:50.506 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.61624.log.gz 2026-02-20T21:28:50.506 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.79118.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62260.log 2026-02-20T21:28:50.506 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.60282.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.admin.79118.log.gz 2026-02-20T21:28:50.506 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63932.log 2026-02-20T21:28:50.506 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.60282.log.gz 2026-02-20T21:28:50.506 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.62260.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.86186.log 2026-02-20T21:28:50.506 INFO:teuthology.orchestra.run.trial199.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.62260.log.gz 2026-02-20T21:28:50.507 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.63932.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86467.log 2026-02-20T21:28:50.507 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.3.86186.log: 19.7% -- replaced with /var/log/ceph/cluster2-client.mirror.3.86186.log.gz 2026-02-20T21:28:50.507 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86601.log 2026-02-20T21:28:50.507 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.86467.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.86467.log.gz 2026-02-20T21:28:50.507 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-mon.a.log 2026-02-20T21:28:50.507 INFO:teuthology.orchestra.run.trial199.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.63932.log.gz 2026-02-20T21:28:50.507 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.86601.log: 27.9% -- replaced with /var/log/ceph/cluster2-client.mirror.86601.log.gz 2026-02-20T21:28:50.507 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79259.log 2026-02-20T21:28:50.508 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75948.log 2026-02-20T21:28:50.508 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.79259.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80819.log 2026-02-20T21:28:50.508 INFO:teuthology.orchestra.run.trial199.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.79259.log.gz 2026-02-20T21:28:50.508 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47509.log 2026-02-20T21:28:50.508 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75948.log: /var/log/ceph/cluster1-client.mirror.80819.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74161.log 2026-02-20T21:28:50.509 INFO:teuthology.orchestra.run.trial199.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.51385.log.gz 2026-02-20T21:28:50.509 INFO:teuthology.orchestra.run.trial199.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.80819.log.gz 2026-02-20T21:28:50.509 INFO:teuthology.orchestra.run.trial199.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.75948.log.gz 2026-02-20T21:28:50.509 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.47509.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47509.log.gz 2026-02-20T21:28:50.509 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86873.log 2026-02-20T21:28:50.509 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.74161.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70855.log 2026-02-20T21:28:50.509 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.86873.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49344.log 2026-02-20T21:28:50.509 INFO:teuthology.orchestra.run.trial199.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.74161.log.gz 2026-02-20T21:28:50.510 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.86873.log.gz 2026-02-20T21:28:50.510 INFO:teuthology.orchestra.run.trial199.stderr:gzip/var/log/ceph/cluster1-client.admin.70855.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88503.log 2026-02-20T21:28:50.510 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.49344.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53415.log 2026-02-20T21:28:50.510 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.70855.log.gz 2026-02-20T21:28:50.510 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.88503.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.49344.log.gz 2026-02-20T21:28:50.510 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62660.log 2026-02-20T21:28:50.510 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.88503.log.gz 2026-02-20T21:28:50.510 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.53415.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61440.log 2026-02-20T21:28:50.511 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.62660.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76789.log 2026-02-20T21:28:50.511 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.62660.log.gz 2026-02-20T21:28:50.511 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.53415.log.gz 2026-02-20T21:28:50.511 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.61440.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88659.log 2026-02-20T21:28:50.511 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.61440.log.gz 2026-02-20T21:28:50.511 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76789.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59896.log 2026-02-20T21:28:50.511 INFO:teuthology.orchestra.run.trial199.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76789.log.gz 2026-02-20T21:28:50.511 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.88659.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74838.log 2026-02-20T21:28:50.511 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.88659.log.gz 2026-02-20T21:28:50.512 INFO:teuthology.orchestra.run.trial199.stderr:gzip/var/log/ceph/cluster1-client.mirror.59896.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83904.log 2026-02-20T21:28:50.512 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.74838.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75329.log 2026-02-20T21:28:50.512 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.59896.log.gz 2026-02-20T21:28:50.512 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.74838.log.gz 2026-02-20T21:28:50.512 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.83904.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66620.log 2026-02-20T21:28:50.512 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.83904.log.gz 2026-02-20T21:28:50.512 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75329.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78411.log 2026-02-20T21:28:50.513 INFO:teuthology.orchestra.run.trial199.stderr: 80.6% -- replaced with /var/log/ceph/cluster2-client.mirror.75329.log.gz 2026-02-20T21:28:50.513 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.66620.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64936.log 2026-02-20T21:28:50.513 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.66620.log.gz 2026-02-20T21:28:50.513 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.78411.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62824.log 2026-02-20T21:28:50.513 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.64936.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83448.log 2026-02-20T21:28:50.513 INFO:teuthology.orchestra.run.trial199.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.64936.log.gz 2026-02-20T21:28:50.513 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.62824.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84211.log 2026-02-20T21:28:50.514 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.62824.log.gz 2026-02-20T21:28:50.514 INFO:teuthology.orchestra.run.trial199.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-client.mirror.78411.log.gz 2026-02-20T21:28:50.514 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.83448.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49877.log 2026-02-20T21:28:50.514 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.83448.log.gz 2026-02-20T21:28:50.514 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.84211.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58116.log 2026-02-20T21:28:50.514 INFO:teuthology.orchestra.run.trial199.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.84211.log.gz 2026-02-20T21:28:50.514 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.49877.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49268.log 2026-02-20T21:28:50.514 INFO:teuthology.orchestra.run.trial199.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.49877.log.gz 2026-02-20T21:28:50.514 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.58116.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54988.log 2026-02-20T21:28:50.515 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.49268.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69836.log 2026-02-20T21:28:50.515 INFO:teuthology.orchestra.run.trial199.stderr: 87.3%/var/log/ceph/cluster1-client.mirror.54988.log: -- replaced with /var/log/ceph/cluster1-client.admin.49268.log.gz 2026-02-20T21:28:50.515 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84188.log 2026-02-20T21:28:50.515 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.58116.log.gz 2026-02-20T21:28:50.515 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.69836.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74181.log 2026-02-20T21:28:50.516 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.84188.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50315.log 2026-02-20T21:28:50.516 INFO:teuthology.orchestra.run.trial199.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54988.log.gz 2026-02-20T21:28:50.516 INFO:teuthology.orchestra.run.trial199.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.84188.log.gz 2026-02-20T21:28:50.516 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.74181.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81948.log 2026-02-20T21:28:50.516 INFO:teuthology.orchestra.run.trial199.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.69836.log.gz 2026-02-20T21:28:50.516 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.50315.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60256.log 2026-02-20T21:28:50.516 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.81948.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.91244.log 2026-02-20T21:28:50.516 INFO:teuthology.orchestra.run.trial199.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74181.log.gz 2026-02-20T21:28:50.516 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.50315.log.gz 2026-02-20T21:28:50.517 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.60256.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64275.log 2026-02-20T21:28:50.517 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81948.log.gz 2026-02-20T21:28:50.517 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.91244.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.91244.log.gz 2026-02-20T21:28:50.517 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82462.log 2026-02-20T21:28:50.517 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.64275.log: gzip -5 --verbose 90.5% -- /var/log/ceph/cluster1-client.admin.56616.log -- replaced with /var/log/ceph/cluster2-client.mirror.60256.log.gz 2026-02-20T21:28:50.517 INFO:teuthology.orchestra.run.trial199.stderr: 2026-02-20T21:28:50.517 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.64275.log.gz 2026-02-20T21:28:50.517 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.82462.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53874.log 2026-02-20T21:28:50.517 INFO:teuthology.orchestra.run.trial199.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.82462.log.gz 2026-02-20T21:28:50.518 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.56616.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67948.log 2026-02-20T21:28:50.518 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.56616.log.gz 2026-02-20T21:28:50.518 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.53874.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76232.log 2026-02-20T21:28:50.518 INFO:teuthology.orchestra.run.trial199.stderr: 80.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53874.log.gz 2026-02-20T21:28:50.518 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.67948.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53778.log 2026-02-20T21:28:50.518 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.67948.log.gz 2026-02-20T21:28:50.518 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76232.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89314.log 2026-02-20T21:28:50.519 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.53778.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48293.log 2026-02-20T21:28:50.519 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.53778.log.gz 2026-02-20T21:28:50.519 INFO:teuthology.orchestra.run.trial199.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.76232.log.gz 2026-02-20T21:28:50.519 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.89314.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63024.log 2026-02-20T21:28:50.519 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.89314.log.gz 2026-02-20T21:28:50.519 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.48293.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62236.log 2026-02-20T21:28:50.519 INFO:teuthology.orchestra.run.trial199.stderr: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48293.log.gz 2026-02-20T21:28:50.519 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.63024.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60813.log 2026-02-20T21:28:50.520 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.63024.log.gz 2026-02-20T21:28:50.520 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.62236.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.90492.log 2026-02-20T21:28:50.520 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62236.log.gz 2026-02-20T21:28:50.520 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.60813.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69615.log 2026-02-20T21:28:50.520 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.60813.log.gz 2026-02-20T21:28:50.520 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.90492.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.90492.log.gz 2026-02-20T21:28:50.520 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61031.log 2026-02-20T21:28:50.520 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.69615.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76253.log 2026-02-20T21:28:50.521 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.61031.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81717.log 2026-02-20T21:28:50.521 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.69615.log.gz 2026-02-20T21:28:50.521 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76253.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65889.log 2026-02-20T21:28:50.521 INFO:teuthology.orchestra.run.trial199.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.76253.log.gz 2026-02-20T21:28:50.521 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.81717.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.61031.log.gz 2026-02-20T21:28:50.521 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83732.log 2026-02-20T21:28:50.521 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.81717.log.gz 2026-02-20T21:28:50.521 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.65889.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84780.log 2026-02-20T21:28:50.522 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.65889.log.gz 2026-02-20T21:28:50.522 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.83732.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87869.log 2026-02-20T21:28:50.522 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.83732.log.gz 2026-02-20T21:28:50.522 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.84780.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86703.log 2026-02-20T21:28:50.522 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.84780.log.gz 2026-02-20T21:28:50.522 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.87869.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61416.log 2026-02-20T21:28:50.522 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.87869.log.gz 2026-02-20T21:28:50.523 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.86703.log: gzip -5 --verbose -- 80.2% /var/log/ceph/cluster1-client.mirror.2.84920.log 2026-02-20T21:28:50.523 INFO:teuthology.orchestra.run.trial199.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.86703.log.gz 2026-02-20T21:28:50.523 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.61416.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48165.log 2026-02-20T21:28:50.523 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61416.log.gz 2026-02-20T21:28:50.523 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.2.84920.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.88953.log 2026-02-20T21:28:50.523 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.48165.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48165.log.gz 2026-02-20T21:28:50.523 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58962.log 2026-02-20T21:28:50.524 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.88953.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67078.log 2026-02-20T21:28:50.524 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.88953.log.gz 2026-02-20T21:28:50.524 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.58962.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65551.log 2026-02-20T21:28:50.524 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.58962.log.gz 2026-02-20T21:28:50.524 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.67078.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68077.log 2026-02-20T21:28:50.524 INFO:teuthology.orchestra.run.trial199.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.2.84920.log.gz 2026-02-20T21:28:50.524 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.67078.log.gz 2026-02-20T21:28:50.525 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.65551.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56723.log 2026-02-20T21:28:50.525 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.65551.log.gz 2026-02-20T21:28:50.525 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.68077.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66763.log 2026-02-20T21:28:50.525 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.68077.log.gz 2026-02-20T21:28:50.525 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.56723.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82191.log 2026-02-20T21:28:50.525 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.66763.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74115.log 2026-02-20T21:28:50.525 INFO:teuthology.orchestra.run.trial199.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.56723.log.gz 2026-02-20T21:28:50.526 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.82191.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.80743.log 2026-02-20T21:28:50.526 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.82191.log.gz 2026-02-20T21:28:50.526 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.74115.log: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.66763.log.gz 2026-02-20T21:28:50.526 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83213.log 2026-02-20T21:28:50.526 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.74115.log.gz 2026-02-20T21:28:50.526 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.80743.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77968.log 2026-02-20T21:28:50.526 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.83213.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.admin.80743.log.gz 2026-02-20T21:28:50.526 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64441.log 2026-02-20T21:28:50.527 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.77968.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70489.log 2026-02-20T21:28:50.527 INFO:teuthology.orchestra.run.trial199.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.83213.log.gz 2026-02-20T21:28:50.527 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.77968.log.gz 2026-02-20T21:28:50.527 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.64441.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85637.log 2026-02-20T21:28:50.527 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.70489.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51472.log 2026-02-20T21:28:50.527 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.70489.log.gz 2026-02-20T21:28:50.527 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.64441.log.gz 2026-02-20T21:28:50.527 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.85637.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85562.log 2026-02-20T21:28:50.528 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.51472.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58063.log 2026-02-20T21:28:50.528 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.51472.log.gz 2026-02-20T21:28:50.528 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.85562.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65337.log 2026-02-20T21:28:50.528 INFO:teuthology.orchestra.run.trial199.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.85637.log.gz 2026-02-20T21:28:50.528 INFO:teuthology.orchestra.run.trial199.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.admin.85562.log.gz 2026-02-20T21:28:50.528 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.58063.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48139.log 2026-02-20T21:28:50.528 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.58063.log.gz 2026-02-20T21:28:50.528 INFO:teuthology.orchestra.run.trial199.stderr:gzip/var/log/ceph/cluster1-client.admin.65337.log: -5 --verbose -- /var/log/ceph/cluster2-client.admin.64050.log 2026-02-20T21:28:50.529 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.48139.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84100.log 2026-02-20T21:28:50.529 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48139.log.gz 2026-02-20T21:28:50.529 INFO:teuthology.orchestra.run.trial199.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.65337.log.gz 2026-02-20T21:28:50.529 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.64050.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63528.log 2026-02-20T21:28:50.529 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.64050.log.gz 2026-02-20T21:28:50.529 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.84100.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50103.log 2026-02-20T21:28:50.529 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.84100.log.gz 2026-02-20T21:28:50.530 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.63528.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55695.log 2026-02-20T21:28:50.530 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.50103.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52412.log 2026-02-20T21:28:50.530 INFO:teuthology.orchestra.run.trial199.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63528.log.gz 2026-02-20T21:28:50.530 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.55695.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50103.log.gz 2026-02-20T21:28:50.530 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89338.log 2026-02-20T21:28:50.530 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.55695.log.gz 2026-02-20T21:28:50.530 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.52412.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75721.log 2026-02-20T21:28:50.531 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.52412.log.gz 2026-02-20T21:28:50.531 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.89338.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61727.log 2026-02-20T21:28:50.531 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.89338.log.gz 2026-02-20T21:28:50.531 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75721.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69886.log 2026-02-20T21:28:50.531 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.61727.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.75721.log.gz 2026-02-20T21:28:50.531 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72621.log 2026-02-20T21:28:50.531 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.61727.log.gz 2026-02-20T21:28:50.531 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.69886.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48882.log 2026-02-20T21:28:50.532 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69886.log.gz 2026-02-20T21:28:50.532 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.72621.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83711.log 2026-02-20T21:28:50.532 INFO:teuthology.orchestra.run.trial199.stderr: 80.5% -- replaced with /var/log/ceph/cluster2-client.mirror.72621.log.gz 2026-02-20T21:28:50.532 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.48882.log: 57.7%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.48882.log.gz -5 2026-02-20T21:28:50.532 INFO:teuthology.orchestra.run.trial199.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.89140.log 2026-02-20T21:28:50.532 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.83711.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50660.log 2026-02-20T21:28:50.532 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.83711.log.gz 2026-02-20T21:28:50.532 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.89140.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78548.log 2026-02-20T21:28:50.533 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.89140.log.gz 2026-02-20T21:28:50.533 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.50660.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66918.log 2026-02-20T21:28:50.533 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.78548.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72719.log 2026-02-20T21:28:50.533 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.78548.log.gz 2026-02-20T21:28:50.533 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.66918.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62585.log 2026-02-20T21:28:50.533 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.66918.log.gz 2026-02-20T21:28:50.533 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.72719.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.54236.log 2026-02-20T21:28:50.534 INFO:teuthology.orchestra.run.trial199.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.72719.log.gz 2026-02-20T21:28:50.534 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.62585.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76152.log 2026-02-20T21:28:50.534 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.3.54236.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.62585.log.gz 2026-02-20T21:28:50.534 INFO:teuthology.orchestra.run.trial199.stderr: 19.7% -- replaced with /var/log/ceph/cluster2-client.mirror.3.54236.log.gz 2026-02-20T21:28:50.534 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76095.log 2026-02-20T21:28:50.534 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76152.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70209.log 2026-02-20T21:28:50.535 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.76095.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58336.log 2026-02-20T21:28:50.535 INFO:teuthology.orchestra.run.trial199.stderr: 90.3% 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.76152.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.76095.log.gz 2026-02-20T21:28:50.535 INFO:teuthology.orchestra.run.trial199.stderr: 2026-02-20T21:28:50.535 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.70209.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57624.log 2026-02-20T21:28:50.535 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.58336.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70791.log 2026-02-20T21:28:50.535 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.57624.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.70209.log.gz 2026-02-20T21:28:50.535 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51728.log 2026-02-20T21:28:50.536 INFO:teuthology.orchestra.run.trial199.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.57624.log.gz 2026-02-20T21:28:50.536 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.70791.log: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.58336.log.gz 2026-02-20T21:28:50.536 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76999.log 2026-02-20T21:28:50.536 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.51728.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73158.log 2026-02-20T21:28:50.536 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76999.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.51728.log.gz 2026-02-20T21:28:50.536 INFO:teuthology.orchestra.run.trial199.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.70791.log.gz 2026-02-20T21:28:50.536 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59110.log 2026-02-20T21:28:50.536 INFO:teuthology.orchestra.run.trial199.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76999.log.gz 2026-02-20T21:28:50.537 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.73158.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76663.log 2026-02-20T21:28:50.537 INFO:teuthology.orchestra.run.trial199.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.50660.log.gz 2026-02-20T21:28:50.537 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.59110.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.73158.log.gz 2026-02-20T21:28:50.537 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73457.log 2026-02-20T21:28:50.537 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76663.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.80521.log 2026-02-20T21:28:50.537 INFO:teuthology.orchestra.run.trial199.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76663.log.gz 2026-02-20T21:28:50.537 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.73457.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.59110.log.gz 2026-02-20T21:28:50.537 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71054.log 2026-02-20T21:28:50.538 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.80521.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85223.log 2026-02-20T21:28:50.538 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.80521.log.gz 2026-02-20T21:28:50.538 INFO:teuthology.orchestra.run.trial199.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.73457.log.gz 2026-02-20T21:28:50.538 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.71054.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90405.log 2026-02-20T21:28:50.538 INFO:teuthology.orchestra.run.trial199.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.71054.log.gz 2026-02-20T21:28:50.538 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.85223.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73230.log 2026-02-20T21:28:50.538 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.85223.log.gz 2026-02-20T21:28:50.538 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.90405.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50429.log 2026-02-20T21:28:50.539 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.73230.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49759.log 2026-02-20T21:28:50.539 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.50429.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.73230.log.gz 2026-02-20T21:28:50.539 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66282.log 2026-02-20T21:28:50.539 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.49759.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58188.log 2026-02-20T21:28:50.539 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.49759.log.gz 2026-02-20T21:28:50.540 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.66282.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57391.log 2026-02-20T21:28:50.540 INFO:teuthology.orchestra.run.trial199.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50429.log.gz 2026-02-20T21:28:50.540 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.66282.log.gz 2026-02-20T21:28:50.540 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.58188.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58242.log 2026-02-20T21:28:50.540 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.58188.log.gz 2026-02-20T21:28:50.540 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.57391.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66111.log 2026-02-20T21:28:50.540 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.57391.log.gz 2026-02-20T21:28:50.540 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.58242.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64477.log 2026-02-20T21:28:50.540 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.58242.log.gz 2026-02-20T21:28:50.541 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.66111.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61081.log 2026-02-20T21:28:50.541 INFO:teuthology.orchestra.run.trial199.stderr: 94.1% -- replaced with /var/log/ceph/cluster1-client.mirror.0.84831.log.gz 2026-02-20T21:28:50.541 INFO:teuthology.orchestra.run.trial199.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.66111.log.gz 2026-02-20T21:28:50.541 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.64477.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64632.log 2026-02-20T21:28:50.541 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.61081.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46970.log 2026-02-20T21:28:50.541 INFO:teuthology.orchestra.run.trial199.stderr: 96.4% -- replaced with /var/log/ceph/cluster1-client.mirror.90405.log.gz 2026-02-20T21:28:50.541 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.61081.log.gz 2026-02-20T21:28:50.542 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.64632.log: gzip -5 --verbose -- 90.4% /var/log/ceph/cluster2-client.mirror.53911.log 2026-02-20T21:28:50.542 INFO:teuthology.orchestra.run.trial199.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.64477.log.gz 2026-02-20T21:28:50.542 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.64632.log.gz 2026-02-20T21:28:50.542 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.46970.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46970.log.gz 2026-02-20T21:28:50.542 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77164.log 2026-02-20T21:28:50.542 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.53911.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.77941.log 2026-02-20T21:28:50.542 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53911.log.gz 2026-02-20T21:28:50.542 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.77164.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77143.log 2026-02-20T21:28:50.543 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.3.77941.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85278.log 2026-02-20T21:28:50.543 INFO:teuthology.orchestra.run.trial199.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.77164.log.gz 2026-02-20T21:28:50.543 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.77143.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83674.log 2026-02-20T21:28:50.543 INFO:teuthology.orchestra.run.trial199.stderr: 88.5%/var/log/ceph/cluster1-client.mirror.85278.log: -- replaced with /var/log/ceph/cluster2-client.mirror.77143.log.gz 2026-02-20T21:28:50.543 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64120.log 2026-02-20T21:28:50.543 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.85278.log.gz 2026-02-20T21:28:50.543 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.83674.log: gzip -5 80.1% --verbose -- -- replaced with /var/log/ceph/cluster2-client.mirror.83674.log.gz 2026-02-20T21:28:50.543 INFO:teuthology.orchestra.run.trial199.stderr: /var/log/ceph/cluster2-client.admin.57678.log 2026-02-20T21:28:50.544 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.64120.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76377.log 2026-02-20T21:28:50.544 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.57678.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56571.log 2026-02-20T21:28:50.544 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.57678.log.gz 2026-02-20T21:28:50.544 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.76377.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.64120.log.gz 2026-02-20T21:28:50.544 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66647.log 2026-02-20T21:28:50.544 INFO:teuthology.orchestra.run.trial199.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.76377.log.gz 2026-02-20T21:28:50.544 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.56571.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46921.log 2026-02-20T21:28:50.544 INFO:teuthology.orchestra.run.trial199.stderr: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.3.77941.log.gz 2026-02-20T21:28:50.545 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.66647.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67972.log 2026-02-20T21:28:50.545 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56571.log.gz 2026-02-20T21:28:50.545 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.66647.log.gz 2026-02-20T21:28:50.545 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.46921.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46921.log.gz 2026-02-20T21:28:50.545 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69991.log 2026-02-20T21:28:50.545 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.67972.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52493.log 2026-02-20T21:28:50.545 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.67972.log.gz 2026-02-20T21:28:50.545 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.69991.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51202.log 2026-02-20T21:28:50.546 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.69991.log.gz 2026-02-20T21:28:50.546 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.52493.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49900.log 2026-02-20T21:28:50.546 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.51202.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.52493.log.gz 2026-02-20T21:28:50.546 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.88796.log 2026-02-20T21:28:50.546 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.51202.log.gz 2026-02-20T21:28:50.546 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.49900.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59184.log 2026-02-20T21:28:50.546 INFO:teuthology.orchestra.run.trial199.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.49900.log.gz 2026-02-20T21:28:50.547 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.88796.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63996.log 2026-02-20T21:28:50.547 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.59184.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.88796.log.gz 2026-02-20T21:28:50.547 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.81181.log 2026-02-20T21:28:50.547 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.63996.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.59184.log.gz 2026-02-20T21:28:50.547 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.80639.log 2026-02-20T21:28:50.547 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.3.81181.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.63996.log.gz 2026-02-20T21:28:50.547 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63906.log 2026-02-20T21:28:50.547 INFO:teuthology.orchestra.run.trial199.stderr: 19.1% -- replaced with /var/log/ceph/cluster1-client.mirror.3.81181.log.gz 2026-02-20T21:28:50.547 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.80639.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.88191.log 2026-02-20T21:28:50.548 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.63906.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.80639.log.gz 2026-02-20T21:28:50.548 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48950.log 2026-02-20T21:28:50.548 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.3.88191.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.63906.log.gz 2026-02-20T21:28:50.548 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79964.log 2026-02-20T21:28:50.548 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.48950.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.91391.log 2026-02-20T21:28:50.548 INFO:teuthology.orchestra.run.trial199.stderr: 69.2% -- replaced with /var/log/ceph/cluster1-client.mirror.48950.log.gz 2026-02-20T21:28:50.548 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.79964.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86897.log 2026-02-20T21:28:50.548 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.79964.log.gz 2026-02-20T21:28:50.548 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.91391.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83233.log 2026-02-20T21:28:50.548 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.91391.log.gz 2026-02-20T21:28:50.548 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.86897.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84375.log 2026-02-20T21:28:50.549 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.83233.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.86897.log.gz 2026-02-20T21:28:50.549 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50396.log 2026-02-20T21:28:50.549 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.83233.log.gz 2026-02-20T21:28:50.549 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.84375.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63475.log 2026-02-20T21:28:50.549 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47264.log 2026-02-20T21:28:50.549 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.50396.log: /var/log/ceph/cluster1-client.admin.63475.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76768.log 2026-02-20T21:28:50.550 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.50396.log.gz 2026-02-20T21:28:50.550 INFO:teuthology.orchestra.run.trial199.stderr: 91.3% -- replaced with /var/log/ceph/cluster1-client.mirror.84375.log.gz 2026-02-20T21:28:50.550 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.47264.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61352.log 2026-02-20T21:28:50.550 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.63475.log.gz 2026-02-20T21:28:50.550 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47264.log.gz 2026-02-20T21:28:50.550 INFO:teuthology.orchestra.run.trial199.stderr: 93.7% -- replaced with /var/log/ceph/cluster1-client.mirror.3.88191.log.gz 2026-02-20T21:28:50.550 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76768.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51980.log 2026-02-20T21:28:50.550 INFO:teuthology.orchestra.run.trial199.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76768.log.gz 2026-02-20T21:28:50.550 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.79313.log 2026-02-20T21:28:50.550 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.61352.log: /var/log/ceph/cluster1-client.admin.51980.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66727.log 2026-02-20T21:28:50.550 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.79313.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73310.log 2026-02-20T21:28:50.551 INFO:teuthology.orchestra.run.trial199.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.51980.log.gz 2026-02-20T21:28:50.551 INFO:teuthology.orchestra.run.trial199.stderr: 87.4%/var/log/ceph/cluster1-client.mirror.66727.log: -- replaced with /var/log/ceph/cluster2-client.admin.79313.log.gz 2026-02-20T21:28:50.551 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60973.log 2026-02-20T21:28:50.551 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.61352.log.gz 2026-02-20T21:28:50.551 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.73310.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.88001.log 2026-02-20T21:28:50.551 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.60973.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49485.log 2026-02-20T21:28:50.551 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.66727.log.gz 2026-02-20T21:28:50.551 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.0.88001.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.60973.log.gz 2026-02-20T21:28:50.551 INFO:teuthology.orchestra.run.trial199.stderr: 19.7%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.0.88001.log.gz -5 2026-02-20T21:28:50.551 INFO:teuthology.orchestra.run.trial199.stderr: --verbose -- /var/log/ceph/cluster2-client.admin.60786.log 2026-02-20T21:28:50.552 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.49485.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67428.log 2026-02-20T21:28:50.552 INFO:teuthology.orchestra.run.trial199.stderr: 68.8% -- replaced with /var/log/ceph/cluster1-client.mirror.49485.log.gz 2026-02-20T21:28:50.552 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.60786.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64578.log 2026-02-20T21:28:50.552 INFO:teuthology.orchestra.run.trial199.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.73310.log.gz 2026-02-20T21:28:50.552 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.60786.log.gz/var/log/ceph/cluster2-client.admin.67428.log: 2026-02-20T21:28:50.552 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63774.log 2026-02-20T21:28:50.552 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.67428.log.gz 2026-02-20T21:28:50.552 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.64578.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57238.log 2026-02-20T21:28:50.552 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.63774.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.64578.log.gz 2026-02-20T21:28:50.552 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75592.log 2026-02-20T21:28:50.552 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.63774.log.gz 2026-02-20T21:28:50.553 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.57238.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81974.log 2026-02-20T21:28:50.553 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75592.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.57238.log.gz 2026-02-20T21:28:50.553 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.88054.log 2026-02-20T21:28:50.553 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.81974.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78602.log 2026-02-20T21:28:50.553 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.81974.log.gz 2026-02-20T21:28:50.553 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.1.88054.log: gzip -5 --verbose 89.5% -- /var/log/ceph/cluster2-client.mirror.50192.log -- replaced with /var/log/ceph/cluster2-client.mirror.75592.log.gz 2026-02-20T21:28:50.553 INFO:teuthology.orchestra.run.trial199.stderr: 2026-02-20T21:28:50.553 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.78602.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54080.log 2026-02-20T21:28:50.553 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.50192.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.78602.log.gz 2026-02-20T21:28:50.553 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82400.log 2026-02-20T21:28:50.554 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.54080.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57319.log 2026-02-20T21:28:50.554 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.54080.log.gz 2026-02-20T21:28:50.554 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.82400.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50192.log.gz 2026-02-20T21:28:50.554 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49145.log 2026-02-20T21:28:50.554 INFO:teuthology.orchestra.run.trial199.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82400.log.gz 2026-02-20T21:28:50.554 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.42955.log 2026-02-20T21:28:50.554 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.57319.log: /var/log/ceph/cluster2-client.mirror.49145.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65662.log 2026-02-20T21:28:50.554 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.57319.log.gz 2026-02-20T21:28:50.555 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49145.log.gz 2026-02-20T21:28:50.555 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.42955.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77184.log 2026-02-20T21:28:50.555 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.42955.log.gz 2026-02-20T21:28:50.555 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.65662.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.77754.log 2026-02-20T21:28:50.555 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65662.log.gz 2026-02-20T21:28:50.555 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.77184.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77206.log 2026-02-20T21:28:50.555 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.0.77754.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72121.log 2026-02-20T21:28:50.555 INFO:teuthology.orchestra.run.trial199.stderr: 19.1% -- replaced with /var/log/ceph/cluster1-client.mirror.0.77754.log.gz 2026-02-20T21:28:50.555 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.77206.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89266.log 2026-02-20T21:28:50.556 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.72121.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70909.log 2026-02-20T21:28:50.556 INFO:teuthology.orchestra.run.trial199.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77206.log.gz 2026-02-20T21:28:50.556 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.72121.log.gz 2026-02-20T21:28:50.556 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.89266.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65578.log 2026-02-20T21:28:50.556 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.89266.log.gz 2026-02-20T21:28:50.556 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.70909.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.77184.log.gz 2026-02-20T21:28:50.556 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83326.log 2026-02-20T21:28:50.556 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.70909.log.gz 2026-02-20T21:28:50.556 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.65578.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56387.log 2026-02-20T21:28:50.557 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.65578.log.gz 2026-02-20T21:28:50.557 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.83326.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83196.log 2026-02-20T21:28:50.557 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.56387.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75907.log 2026-02-20T21:28:50.557 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.83326.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.56387.log.gz 2026-02-20T21:28:50.557 INFO:teuthology.orchestra.run.trial199.stderr: 2026-02-20T21:28:50.557 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51701.log 2026-02-20T21:28:50.557 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.83196.log: 80.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83196.log.gz 2026-02-20T21:28:50.557 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75907.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88527.log 2026-02-20T21:28:50.557 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.51701.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79477.log 2026-02-20T21:28:50.557 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.51701.log.gz 2026-02-20T21:28:50.557 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.88527.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50242.log 2026-02-20T21:28:50.558 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.88527.log.gz 2026-02-20T21:28:50.558 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.79477.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78622.log 2026-02-20T21:28:50.558 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.50242.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65391.log 2026-02-20T21:28:50.558 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.50242.log.gz 2026-02-20T21:28:50.558 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.78622.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.79477.log.gz 2026-02-20T21:28:50.558 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59291.log 2026-02-20T21:28:50.558 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.65391.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85521.log 2026-02-20T21:28:50.558 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.59291.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.65391.log.gz 2026-02-20T21:28:50.558 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60684.log 2026-02-20T21:28:50.559 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.85521.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67276.log 2026-02-20T21:28:50.559 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.60684.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.59291.log.gz 2026-02-20T21:28:50.559 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82325.log 2026-02-20T21:28:50.559 INFO:teuthology.orchestra.run.trial199.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.85521.log.gz 2026-02-20T21:28:50.559 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60684.log.gz 2026-02-20T21:28:50.559 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.67276.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68373.log 2026-02-20T21:28:50.559 INFO:teuthology.orchestra.run.trial199.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.78622.log.gz 2026-02-20T21:28:50.559 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.82325.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.67276.log.gz 2026-02-20T21:28:50.559 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.48602.log 2026-02-20T21:28:50.560 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.68373.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74718.log 2026-02-20T21:28:50.560 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.68373.log.gz 2026-02-20T21:28:50.560 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.2.48602.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.89017.log 2026-02-20T21:28:50.560 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.74718.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70765.log 2026-02-20T21:28:50.560 INFO:teuthology.orchestra.run.trial199.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.82325.log.gz 2026-02-20T21:28:50.560 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.74718.log.gz 2026-02-20T21:28:50.560 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.89017.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46872.log 2026-02-20T21:28:50.560 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.89017.log.gz 2026-02-20T21:28:50.560 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.70765.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61571.log 2026-02-20T21:28:50.561 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.46872.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75120.log 2026-02-20T21:28:50.561 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.70765.log.gz 2026-02-20T21:28:50.561 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46872.log.gz 2026-02-20T21:28:50.561 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.61571.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71890.log 2026-02-20T21:28:50.561 INFO:teuthology.orchestra.run.trial199.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75907.log.gz 2026-02-20T21:28:50.561 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75120.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71653.log 2026-02-20T21:28:50.561 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.75120.log.gz 2026-02-20T21:28:50.561 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.71890.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.61571.log.gz 2026-02-20T21:28:50.561 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65737.log 2026-02-20T21:28:50.562 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.71890.log.gz 2026-02-20T21:28:50.562 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.71653.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49448.log 2026-02-20T21:28:50.562 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.65737.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.71653.log.gz 2026-02-20T21:28:50.562 INFO:teuthology.orchestra.run.trial199.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.2.48602.log.gz 2026-02-20T21:28:50.562 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85928.log 2026-02-20T21:28:50.562 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.49448.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.65737.log.gz 2026-02-20T21:28:50.562 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53170.log 2026-02-20T21:28:50.562 INFO:teuthology.orchestra.run.trial199.stderr: 55.3% -- replaced with /var/log/ceph/cluster2-client.mirror.49448.log.gz 2026-02-20T21:28:50.562 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.85928.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69589.log 2026-02-20T21:28:50.562 INFO:teuthology.orchestra.run.trial199.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.admin.85928.log.gz 2026-02-20T21:28:50.562 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.53170.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62420.log 2026-02-20T21:28:50.563 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.69589.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.80120.log 2026-02-20T21:28:50.563 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.62420.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.69589.log.gz 2026-02-20T21:28:50.563 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63155.log 2026-02-20T21:28:50.563 INFO:teuthology.orchestra.run.trial199.stderr: 87.4%/var/log/ceph/cluster1-client.mirror.2.80120.log: -- replaced with /var/log/ceph/cluster1-client.admin.62420.log.gz 2026-02-20T21:28:50.563 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65916.log 2026-02-20T21:28:50.563 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.63155.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58989.log 2026-02-20T21:28:50.563 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.65916.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86626.log 2026-02-20T21:28:50.563 INFO:teuthology.orchestra.run.trial199.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53170.log.gz 2026-02-20T21:28:50.564 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.65916.log.gz 2026-02-20T21:28:50.564 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.58989.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.63155.log.gz 2026-02-20T21:28:50.564 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74137.log 2026-02-20T21:28:50.564 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.58989.log.gz 2026-02-20T21:28:50.564 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.86626.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82667.log 2026-02-20T21:28:50.564 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.86626.log.gz 2026-02-20T21:28:50.564 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.74137.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86046.log 2026-02-20T21:28:50.564 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.82667.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.74137.log.gz 2026-02-20T21:28:50.564 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.78378.log 2026-02-20T21:28:50.564 INFO:teuthology.orchestra.run.trial199.stderr: 80.4% -- replaced with /var/log/ceph/cluster2-client.mirror.82667.log.gz 2026-02-20T21:28:50.564 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.86046.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48122.log 2026-02-20T21:28:50.565 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.78378.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48434.log 2026-02-20T21:28:50.565 INFO:teuthology.orchestra.run.trial199.stderr: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.2.80120.log.gz 2026-02-20T21:28:50.565 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.48122.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.77862.log 2026-02-20T21:28:50.565 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.78378.log.gz 2026-02-20T21:28:50.565 INFO:teuthology.orchestra.run.trial199.stderr: 56.5% -- replaced with /var/log/ceph/cluster1-client.admin.48122.log.gz 2026-02-20T21:28:50.565 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.48434.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88829.log 2026-02-20T21:28:50.565 INFO:teuthology.orchestra.run.trial199.stderr: 53.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48434.log.gz 2026-02-20T21:28:50.565 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.2.77862.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58890.log 2026-02-20T21:28:50.565 INFO:teuthology.orchestra.run.trial199.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.86046.log.gz 2026-02-20T21:28:50.566 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.88829.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73902.log 2026-02-20T21:28:50.566 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.58890.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83809.log 2026-02-20T21:28:50.566 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.73902.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68595.log 2026-02-20T21:28:50.566 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.83809.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85810.log 2026-02-20T21:28:50.566 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.58890.log.gz 2026-02-20T21:28:50.566 INFO:teuthology.orchestra.run.trial199.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.73902.log.gz 2026-02-20T21:28:50.566 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.68595.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66406.log 2026-02-20T21:28:50.566 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.68595.log.gz 2026-02-20T21:28:50.566 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.85810.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75201.log 2026-02-20T21:28:50.567 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.66406.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.85810.log.gz 2026-02-20T21:28:50.567 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47930.log 2026-02-20T21:28:50.567 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75201.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47913.log 2026-02-20T21:28:50.567 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.47930.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66944.log 2026-02-20T21:28:50.567 INFO:teuthology.orchestra.run.trial199.stderr: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.47930.log.gz 2026-02-20T21:28:50.567 INFO:teuthology.orchestra.run.trial199.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.2.77862.log.gz 2026-02-20T21:28:50.567 INFO:teuthology.orchestra.run.trial199.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.66406.log.gz 2026-02-20T21:28:50.567 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.47913.log: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.47913.log.gz 2026-02-20T21:28:50.568 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68451.log 2026-02-20T21:28:50.568 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.66944.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.91048.log 2026-02-20T21:28:50.568 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.68451.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64862.log 2026-02-20T21:28:50.568 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.68451.log.gz 2026-02-20T21:28:50.568 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.91048.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.66944.log.gz 2026-02-20T21:28:50.568 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69535.log 2026-02-20T21:28:50.568 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.91048.log.gz 2026-02-20T21:28:50.568 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.64862.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89120.log 2026-02-20T21:28:50.568 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.69535.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78828.log 2026-02-20T21:28:50.569 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.69535.log.gz 2026-02-20T21:28:50.569 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.89120.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71723.log 2026-02-20T21:28:50.569 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.78828.log: 90.6%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.64862.log.gz -5 2026-02-20T21:28:50.569 INFO:teuthology.orchestra.run.trial199.stderr: --verbose -- /var/log/ceph/cluster2-client.admin.67051.log 2026-02-20T21:28:50.569 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78828.log.gz 2026-02-20T21:28:50.569 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89120.log.gz 2026-02-20T21:28:50.569 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.71723.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60105.log 2026-02-20T21:28:50.569 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.67051.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62559.log 2026-02-20T21:28:50.569 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.60105.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.67051.log.gz 2026-02-20T21:28:50.570 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82507.log 2026-02-20T21:28:50.570 INFO:teuthology.orchestra.run.trial199.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.71723.log.gz 2026-02-20T21:28:50.570 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.60105.log.gz 2026-02-20T21:28:50.570 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.62559.log: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.83809.log.gz 2026-02-20T21:28:50.570 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86576.log 2026-02-20T21:28:50.570 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.82507.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62092.log 2026-02-20T21:28:50.570 INFO:teuthology.orchestra.run.trial199.stderr: 86.4% -- replaced with /var/log/ceph/cluster1-client.mirror.82507.log.gz 2026-02-20T21:28:50.570 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.86576.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59042.log 2026-02-20T21:28:50.570 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.86576.log.gz 2026-02-20T21:28:50.570 INFO:teuthology.orchestra.run.trial199.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.62559.log.gz 2026-02-20T21:28:50.571 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.62092.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52340.log 2026-02-20T21:28:50.571 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.62092.log.gz 2026-02-20T21:28:50.571 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.59042.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51226.log 2026-02-20T21:28:50.571 INFO:teuthology.orchestra.run.trial199.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75201.log.gz 2026-02-20T21:28:50.571 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.52340.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.89704.log 2026-02-20T21:28:50.571 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.59042.log.gz 2026-02-20T21:28:50.571 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.51226.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80796.log 2026-02-20T21:28:50.571 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.52340.log.gz 2026-02-20T21:28:50.571 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.0.89704.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.51226.log.gz 2026-02-20T21:28:50.571 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.79340.log 2026-02-20T21:28:50.571 INFO:teuthology.orchestra.run.trial199.stderr: 18.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.89704.log.gz 2026-02-20T21:28:50.572 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.80796.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.90950.log 2026-02-20T21:28:50.572 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.79340.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72436.log 2026-02-20T21:28:50.572 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.90950.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80776.log 2026-02-20T21:28:50.572 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.90950.log.gz 2026-02-20T21:28:50.572 INFO:teuthology.orchestra.run.trial199.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.admin.79340.log.gz 2026-02-20T21:28:50.572 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.72436.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.87026.log 2026-02-20T21:28:50.572 INFO:teuthology.orchestra.run.trial199.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.80796.log.gz 2026-02-20T21:28:50.573 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.80776.log: 88.4%gzip -- replaced with /var/log/ceph/cluster2-client.admin.72436.log.gz 2026-02-20T21:28:50.573 INFO:teuthology.orchestra.run.trial199.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.81117.log 2026-02-20T21:28:50.573 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.87026.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82776.log 2026-02-20T21:28:50.573 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.0.81117.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.87026.log.gz 2026-02-20T21:28:50.573 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75016.log 2026-02-20T21:28:50.573 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.82776.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63573.log 2026-02-20T21:28:50.573 INFO:teuthology.orchestra.run.trial199.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.80776.log.gz 2026-02-20T21:28:50.573 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.75016.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.84822.log 2026-02-20T21:28:50.573 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.63573.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83928.log 2026-02-20T21:28:50.573 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.63573.log.gz 2026-02-20T21:28:50.574 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.0.84822.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71572.log 2026-02-20T21:28:50.574 INFO:teuthology.orchestra.run.trial199.stderr: 19.7% -- replaced with /var/log/ceph/cluster1-client.mirror.0.84822.log.gz 2026-02-20T21:28:50.574 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.83928.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51422.log 2026-02-20T21:28:50.574 INFO:teuthology.orchestra.run.trial199.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.75016.log.gz 2026-02-20T21:28:50.574 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.71572.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.83928.log.gz 2026-02-20T21:28:50.574 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83856.log 2026-02-20T21:28:50.574 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.71572.log.gz 2026-02-20T21:28:50.574 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.51422.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78876.log 2026-02-20T21:28:50.575 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.83856.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59765.log 2026-02-20T21:28:50.575 INFO:teuthology.orchestra.run.trial199.stderr: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.0.81117.log.gz 2026-02-20T21:28:50.575 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.78876.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76335.log 2026-02-20T21:28:50.575 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.83856.log.gz 2026-02-20T21:28:50.575 INFO:teuthology.orchestra.run.trial199.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51422.log.gz 2026-02-20T21:28:50.575 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.78876.log.gz 2026-02-20T21:28:50.575 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.59765.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60330.log 2026-02-20T21:28:50.575 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76335.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.59765.log.gz 2026-02-20T21:28:50.575 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62740.log 2026-02-20T21:28:50.575 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.60330.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.76335.log.gz 2026-02-20T21:28:50.575 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55920.log 2026-02-20T21:28:50.575 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.62740.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.60330.log.gz 2026-02-20T21:28:50.576 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.88114.log 2026-02-20T21:28:50.576 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.55920.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.78927.log 2026-02-20T21:28:50.576 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.2.88114.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53019.log 87.4% 2026-02-20T21:28:50.576 INFO:teuthology.orchestra.run.trial199.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.55920.log.gz 2026-02-20T21:28:50.576 INFO:teuthology.orchestra.run.trial199.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62740.log.gz 2026-02-20T21:28:50.576 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.78927.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.79768.log 2026-02-20T21:28:50.576 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.53019.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55097.log 2026-02-20T21:28:50.576 INFO:teuthology.orchestra.run.trial199.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.78927.log.gz 2026-02-20T21:28:50.576 INFO:teuthology.orchestra.run.trial199.stderr: 87.4%/var/log/ceph/cluster2-client.admin.79768.log: -- replaced with /var/log/ceph/cluster1-client.admin.53019.log.gz 2026-02-20T21:28:50.577 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57891.log 2026-02-20T21:28:50.577 INFO:teuthology.orchestra.run.trial199.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.79768.log.gz 2026-02-20T21:28:50.577 INFO:teuthology.orchestra.run.trial199.stderr: 94.5%/var/log/ceph/cluster2-client.admin.55097.log: -- replaced with /var/log/ceph/cluster2-client.mirror.82776.log.gz 2026-02-20T21:28:50.577 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88455.log 2026-02-20T21:28:50.577 INFO:teuthology.orchestra.run.trial199.stderr: 87.6%/var/log/ceph/cluster2-client.mirror.57891.log: -- replaced with /var/log/ceph/cluster2-client.admin.55097.log.gz 2026-02-20T21:28:50.577 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76192.log 2026-02-20T21:28:50.577 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.88455.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51340.log 2026-02-20T21:28:50.577 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.88455.log.gz/var/log/ceph/cluster2-client.mirror.76192.log: 2026-02-20T21:28:50.577 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87052.log 2026-02-20T21:28:50.577 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.57891.log.gz 2026-02-20T21:28:50.578 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.51340.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64076.log 2026-02-20T21:28:50.578 INFO:teuthology.orchestra.run.trial199.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.76192.log.gz 2026-02-20T21:28:50.578 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.87052.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85950.log 2026-02-20T21:28:50.578 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.64076.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52160.log 2026-02-20T21:28:50.578 INFO:teuthology.orchestra.run.trial199.stderr: 93.8% -- replaced with /var/log/ceph/cluster1-client.mirror.2.88114.log.gz 2026-02-20T21:28:50.578 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.85950.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61913.log 2026-02-20T21:28:50.578 INFO:teuthology.orchestra.run.trial199.stderr: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.51340.log.gz 2026-02-20T21:28:50.578 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.85950.log.gz 2026-02-20T21:28:50.579 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.52160.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79592.log 2026-02-20T21:28:50.579 INFO:teuthology.orchestra.run.trial199.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64076.log.gz 2026-02-20T21:28:50.579 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.52160.log.gz 2026-02-20T21:28:50.579 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.61913.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66087.log 2026-02-20T21:28:50.579 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.61913.log.gz 2026-02-20T21:28:50.579 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.79592.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86999.log 2026-02-20T21:28:50.579 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.66087.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48967.log 2026-02-20T21:28:50.579 INFO:teuthology.orchestra.run.trial199.stderr: 87.3%/var/log/ceph/cluster2-client.admin.86999.log: -- replaced with /var/log/ceph/cluster2-client.mirror.66087.log.gz 2026-02-20T21:28:50.579 INFO:teuthology.orchestra.run.trial199.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.79592.log.gz 2026-02-20T21:28:50.579 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48451.log 2026-02-20T21:28:50.580 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.48967.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76132.log 2026-02-20T21:28:50.580 INFO:teuthology.orchestra.run.trial199.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.86999.log.gz 2026-02-20T21:28:50.580 INFO:teuthology.orchestra.run.trial199.stderr: 80.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48967.log.gz 2026-02-20T21:28:50.580 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.48451.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.45879.log 2026-02-20T21:28:50.580 INFO:teuthology.orchestra.run.trial199.stderr: 61.6% -- replaced with /var/log/ceph/cluster2-client.admin.48451.log.gz 2026-02-20T21:28:50.580 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76132.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.90516.log 2026-02-20T21:28:50.580 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.45879.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.80198.log 2026-02-20T21:28:50.580 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.45879.log.gz 2026-02-20T21:28:50.580 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.90516.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67890.log 2026-02-20T21:28:50.580 INFO:teuthology.orchestra.run.trial199.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.76132.log.gz 2026-02-20T21:28:50.580 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.90516.log.gz 2026-02-20T21:28:50.580 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.3.80198.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78646.log 2026-02-20T21:28:50.581 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.67890.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59818.log 2026-02-20T21:28:50.581 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.78646.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.80498.log 2026-02-20T21:28:50.581 INFO:teuthology.orchestra.run.trial199.stderr: 80.4% -- replaced with /var/log/ceph/cluster2-client.mirror.78646.log.gz 2026-02-20T21:28:50.581 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.59818.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75346.log 2026-02-20T21:28:50.581 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67890.log.gz 2026-02-20T21:28:50.581 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.80498.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89160.log 2026-02-20T21:28:50.581 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75346.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.80498.log.gz 2026-02-20T21:28:50.581 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63315.log 2026-02-20T21:28:50.581 INFO:teuthology.orchestra.run.trial199.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.59818.log.gz 2026-02-20T21:28:50.582 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.89160.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84512.log 2026-02-20T21:28:50.582 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.75346.log.gz 2026-02-20T21:28:50.582 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.63315.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55995.log 2026-02-20T21:28:50.582 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.84512.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60658.log 2026-02-20T21:28:50.582 INFO:teuthology.orchestra.run.trial199.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.3.80198.log.gz 2026-02-20T21:28:50.582 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.84512.log.gz 2026-02-20T21:28:50.582 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.55995.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.63315.log.gz 2026-02-20T21:28:50.582 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.91146.log 2026-02-20T21:28:50.582 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.55995.log.gz 2026-02-20T21:28:50.583 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.60658.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55350.log 2026-02-20T21:28:50.583 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.91146.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81602.log 2026-02-20T21:28:50.583 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.91146.log.gz 2026-02-20T21:28:50.583 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.55350.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75266.log 2026-02-20T21:28:50.583 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.55350.log.gz 2026-02-20T21:28:50.583 INFO:teuthology.orchestra.run.trial199.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.60658.log.gz 2026-02-20T21:28:50.583 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.81602.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65444.log 2026-02-20T21:28:50.583 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.75266.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65263.log 2026-02-20T21:28:50.584 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.75266.log.gz 2026-02-20T21:28:50.584 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.65444.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.81602.log.gz 2026-02-20T21:28:50.584 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61753.log 2026-02-20T21:28:50.584 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.65263.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52893.log 2026-02-20T21:28:50.584 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.61753.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.65444.log.gz 2026-02-20T21:28:50.584 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82552.log 2026-02-20T21:28:50.584 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.52893.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81581.log 2026-02-20T21:28:50.584 INFO:teuthology.orchestra.run.trial199.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.65263.log.gz 2026-02-20T21:28:50.584 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.61753.log.gz 2026-02-20T21:28:50.584 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.82552.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.52893.log.gz 2026-02-20T21:28:50.585 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69714.log 2026-02-20T21:28:50.585 INFO:teuthology.orchestra.run.trial199.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.82552.log.gz 2026-02-20T21:28:50.585 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.81581.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61326.log 2026-02-20T21:28:50.585 INFO:teuthology.orchestra.run.trial199.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.81581.log.gz 2026-02-20T21:28:50.585 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.69714.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66561.log 2026-02-20T21:28:50.585 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.69714.log.gz 2026-02-20T21:28:50.585 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.61326.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71773.log 2026-02-20T21:28:50.585 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.66561.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.61326.log.gz 2026-02-20T21:28:50.586 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56258.log 2026-02-20T21:28:50.586 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.66561.log.gz 2026-02-20T21:28:50.586 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.71773.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59136.log 2026-02-20T21:28:50.586 INFO:teuthology.orchestra.run.trial199.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.89160.log.gz 2026-02-20T21:28:50.586 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.71773.log.gz 2026-02-20T21:28:50.586 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.56258.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58161.log 2026-02-20T21:28:50.586 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.56258.log.gz 2026-02-20T21:28:50.586 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.59136.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63048.log 2026-02-20T21:28:50.586 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.59136.log.gz 2026-02-20T21:28:50.586 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.58161.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56316.log 2026-02-20T21:28:50.587 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.63048.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.58161.log.gz 2026-02-20T21:28:50.587 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73825.log 2026-02-20T21:28:50.587 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.56316.log: 88.5%gzip -- replaced with /var/log/ceph/cluster1-client.admin.63048.log.gz 2026-02-20T21:28:50.587 INFO:teuthology.orchestra.run.trial199.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.48603.log 2026-02-20T21:28:50.587 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.73825.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53753.log 2026-02-20T21:28:50.587 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.3.48603.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72463.log 2026-02-20T21:28:50.587 INFO:teuthology.orchestra.run.trial199.stderr: 19.7% -- replaced with /var/log/ceph/cluster1-client.mirror.3.48603.log.gz 2026-02-20T21:28:50.587 INFO:teuthology.orchestra.run.trial199.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.73825.log.gz 2026-02-20T21:28:50.587 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.53753.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.53753.log.gz 2026-02-20T21:28:50.587 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79070.log 2026-02-20T21:28:50.588 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.72463.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69484.log 2026-02-20T21:28:50.588 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.79070.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.89819.log 2026-02-20T21:28:50.588 INFO:teuthology.orchestra.run.trial199.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.admin.72463.log.gz 2026-02-20T21:28:50.588 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.69484.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.79070.log.gz 2026-02-20T21:28:50.588 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72382.log 2026-02-20T21:28:50.588 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.69484.log.gz 2026-02-20T21:28:50.588 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.3.89819.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59238.log 2026-02-20T21:28:50.588 INFO:teuthology.orchestra.run.trial199.stderr: 19.7% -- replaced with /var/log/ceph/cluster1-client.mirror.3.89819.log.gz 2026-02-20T21:28:50.589 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.72382.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62904.log 2026-02-20T21:28:50.589 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.59238.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.72382.log.gz 2026-02-20T21:28:50.589 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.89714.log 2026-02-20T21:28:50.589 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.62904.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.59238.log.gz 2026-02-20T21:28:50.589 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.45974.log 2026-02-20T21:28:50.589 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.1.89714.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70711.log 2026-02-20T21:28:50.589 INFO:teuthology.orchestra.run.trial199.stderr: 18.1% -- replaced with /var/log/ceph/cluster1-client.mirror.1.89714.log.gz 2026-02-20T21:28:50.589 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.45974.log: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.62904.log.gz 2026-02-20T21:28:50.589 INFO:teuthology.orchestra.run.trial199.stderr: 80.1% -- replaced with /var/log/ceph/cluster2-client.admin.45974.log.gz 2026-02-20T21:28:50.590 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68202.log 2026-02-20T21:28:50.590 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.70711.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64503.log 2026-02-20T21:28:50.590 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.70711.log.gz 2026-02-20T21:28:50.590 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.68202.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90427.log 2026-02-20T21:28:50.590 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.64503.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.68202.log.gz 2026-02-20T21:28:50.590 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78324.log 2026-02-20T21:28:50.590 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.64503.log.gz 2026-02-20T21:28:50.591 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.90427.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60051.log 2026-02-20T21:28:50.591 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.78324.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88849.log 2026-02-20T21:28:50.591 INFO:teuthology.orchestra.run.trial199.stderr: 94.2% 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56316.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.90427.log.gz 2026-02-20T21:28:50.591 INFO:teuthology.orchestra.run.trial199.stderr: 2026-02-20T21:28:50.591 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.60051.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68648.log 2026-02-20T21:28:50.591 INFO:teuthology.orchestra.run.trial199.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.admin.78324.log.gz 2026-02-20T21:28:50.591 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.88849.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67650.log 2026-02-20T21:28:50.591 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.68648.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87826.log 2026-02-20T21:28:50.592 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.60051.log.gz 2026-02-20T21:28:50.592 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.67650.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74386.log 2026-02-20T21:28:50.592 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.67650.log.gz 2026-02-20T21:28:50.592 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.87826.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47879.log 2026-02-20T21:28:50.592 INFO:teuthology.orchestra.run.trial199.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68648.log.gz 2026-02-20T21:28:50.592 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.74386.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63000.log 2026-02-20T21:28:50.592 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.47879.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69099.log 2026-02-20T21:28:50.592 INFO:teuthology.orchestra.run.trial199.stderr: 70.5% -- replaced with /var/log/ceph/cluster1-client.admin.47879.log.gz 2026-02-20T21:28:50.592 INFO:teuthology.orchestra.run.trial199.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.87826.log.gz 2026-02-20T21:28:50.593 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.63000.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.91195.log 2026-02-20T21:28:50.593 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.63000.log.gz 2026-02-20T21:28:50.593 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.69099.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63075.log 2026-02-20T21:28:50.593 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.69099.log.gz 2026-02-20T21:28:50.593 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.91195.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57498.log 2026-02-20T21:28:50.593 INFO:teuthology.orchestra.run.trial199.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.74386.log.gz 2026-02-20T21:28:50.593 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.91195.log.gz 2026-02-20T21:28:50.593 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.63075.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.54234.log 2026-02-20T21:28:50.593 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.63075.log.gz 2026-02-20T21:28:50.593 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.57498.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74049.log 2026-02-20T21:28:50.594 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.2.54234.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75555.log 2026-02-20T21:28:50.594 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.74049.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67677.log 2026-02-20T21:28:50.594 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.57498.log.gz 2026-02-20T21:28:50.594 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.75555.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59485.log 2026-02-20T21:28:50.594 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.75555.log.gz 2026-02-20T21:28:50.594 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.67677.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86026.log 2026-02-20T21:28:50.594 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.67677.log.gz 2026-02-20T21:28:50.595 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74651.log 2026-02-20T21:28:50.595 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.59485.log: /var/log/ceph/cluster2-client.mirror.86026.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56082.log 2026-02-20T21:28:50.595 INFO:teuthology.orchestra.run.trial199.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.74049.log.gz 2026-02-20T21:28:50.595 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.74651.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.86026.log.gz 2026-02-20T21:28:50.595 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75632.log 2026-02-20T21:28:50.595 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77307.log 2026-02-20T21:28:50.596 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.56082.log: /var/log/ceph/cluster2-client.mirror.75632.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55376.log 2026-02-20T21:28:50.596 INFO:teuthology.orchestra.run.trial199.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59485.log.gz 2026-02-20T21:28:50.596 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.74651.log.gz 2026-02-20T21:28:50.596 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.77307.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70828.log 2026-02-20T21:28:50.596 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.55376.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50640.log 2026-02-20T21:28:50.596 INFO:teuthology.orchestra.run.trial199.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56082.log.gz 2026-02-20T21:28:50.596 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.70828.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49468.log 2026-02-20T21:28:50.597 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.70828.log.gz 2026-02-20T21:28:50.597 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.50640.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.55376.log.gz 2026-02-20T21:28:50.597 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67623.log 2026-02-20T21:28:50.597 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50640.log.gz 2026-02-20T21:28:50.597 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.49468.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71836.log 2026-02-20T21:28:50.597 INFO:teuthology.orchestra.run.trial199.stderr: 69.2% -- replaced with /var/log/ceph/cluster2-client.mirror.49468.log.gz 2026-02-20T21:28:50.597 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.67623.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88575.log 2026-02-20T21:28:50.597 INFO:teuthology.orchestra.run.trial199.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.67623.log.gz 2026-02-20T21:28:50.598 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.71836.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74344.log 2026-02-20T21:28:50.598 INFO:teuthology.orchestra.run.trial199.stderr: 94.8% -- replaced with /var/log/ceph/cluster1-client.mirror.1.88054.log.gz 2026-02-20T21:28:50.598 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.71836.log.gz 2026-02-20T21:28:50.598 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83305.log 2026-02-20T21:28:50.598 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.88575.log: /var/log/ceph/cluster1-client.mirror.74344.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.88575.log.gz 2026-02-20T21:28:50.598 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75512.log 2026-02-20T21:28:50.598 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.74344.log.gz 2026-02-20T21:28:50.599 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.83305.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57292.log 2026-02-20T21:28:50.599 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.75512.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79497.log 2026-02-20T21:28:50.599 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.75512.log.gz 2026-02-20T21:28:50.599 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.57292.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65604.log 2026-02-20T21:28:50.599 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.79497.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.57292.log.gz 2026-02-20T21:28:50.599 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84427.log 2026-02-20T21:28:50.600 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.79497.log.gz 2026-02-20T21:28:50.600 INFO:teuthology.orchestra.run.trial199.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75632.log.gz 2026-02-20T21:28:50.600 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.65604.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.77307.log.gz 2026-02-20T21:28:50.600 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67575.log 2026-02-20T21:28:50.600 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52721.log 2026-02-20T21:28:50.600 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.84427.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65604.log.gz 2026-02-20T21:28:50.600 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.67575.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59426.log 2026-02-20T21:28:50.600 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.84427.log.gz 2026-02-20T21:28:50.601 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.67575.log.gz 2026-02-20T21:28:50.601 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.52721.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75288.log 2026-02-20T21:28:50.601 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.52721.log.gz 2026-02-20T21:28:50.601 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.59426.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78663.log 2026-02-20T21:28:50.601 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.75288.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.54677.log 2026-02-20T21:28:50.601 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.59426.log.gz 2026-02-20T21:28:50.601 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.75288.log.gz 2026-02-20T21:28:50.601 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85837.log 2026-02-20T21:28:50.601 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.54677.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.54171.log 2026-02-20T21:28:50.601 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.54677.log.gz 2026-02-20T21:28:50.602 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5/var/log/ceph/cluster1-client.admin.85837.log: --verbose -- /var/log/ceph/cluster2-client.admin.48417.log 2026-02-20T21:28:50.602 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.78663.log: /var/log/ceph/cluster2-client.mirror.1.54171.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.85837.log.gz 2026-02-20T21:28:50.602 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47796.log 2026-02-20T21:28:50.602 INFO:teuthology.orchestra.run.trial199.stderr: 19.1% -- replaced with /var/log/ceph/cluster2-client.mirror.1.54171.log.gz 2026-02-20T21:28:50.602 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.48417.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71939.log 2026-02-20T21:28:50.602 INFO:teuthology.orchestra.run.trial199.stderr: 56.9% -- replaced with /var/log/ceph/cluster2-client.admin.48417.log.gz 2026-02-20T21:28:50.602 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.47796.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.78663.log.gz 2026-02-20T21:28:50.602 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75865.log 2026-02-20T21:28:50.602 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47796.log.gz 2026-02-20T21:28:50.603 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.71939.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54597.log 2026-02-20T21:28:50.603 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.71939.log.gz 2026-02-20T21:28:50.603 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75865.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61105.log 2026-02-20T21:28:50.603 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.54597.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68622.log 2026-02-20T21:28:50.603 INFO:teuthology.orchestra.run.trial199.stderr: 89.6% 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.75865.log.gz -- replaced with /var/log/ceph/cluster1-client.admin.54597.log.gz 2026-02-20T21:28:50.603 INFO:teuthology.orchestra.run.trial199.stderr: 2026-02-20T21:28:50.603 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.61105.log: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.83305.log.gz 2026-02-20T21:28:50.603 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72894.log 2026-02-20T21:28:50.603 INFO:teuthology.orchestra.run.trial199.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.61105.log.gz 2026-02-20T21:28:50.604 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69964.log 2026-02-20T21:28:50.604 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.68622.log: /var/log/ceph/cluster1-client.mirror.72894.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85999.log 2026-02-20T21:28:50.604 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.68622.log.gz 2026-02-20T21:28:50.604 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.72894.log.gz 2026-02-20T21:28:50.604 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.69964.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.89362.log 2026-02-20T21:28:50.604 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.69964.log.gz 2026-02-20T21:28:50.604 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.85999.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59586.log 2026-02-20T21:28:50.604 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.85999.log.gz 2026-02-20T21:28:50.604 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.89362.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60759.log 2026-02-20T21:28:50.604 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.59586.log: 87.4%gzip -- replaced with /var/log/ceph/cluster1-client.admin.89362.log.gz -5 2026-02-20T21:28:50.605 INFO:teuthology.orchestra.run.trial199.stderr: --verbose -- /var/log/ceph/cluster1-client.admin.57784.log 2026-02-20T21:28:50.605 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.60759.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.59586.log.gz 2026-02-20T21:28:50.605 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.77807.log 2026-02-20T21:28:50.605 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.60759.log.gz 2026-02-20T21:28:50.605 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.57784.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60839.log 2026-02-20T21:28:50.605 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.1.77807.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.57784.log.gz 2026-02-20T21:28:50.605 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70684.log 2026-02-20T21:28:50.605 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.60839.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74450.log 2026-02-20T21:28:50.605 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.70684.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66534.log 2026-02-20T21:28:50.606 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.74450.log: 89.9% 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.60839.log.gz 2026-02-20T21:28:50.606 INFO:teuthology.orchestra.run.trial199.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.70684.log.gz 2026-02-20T21:28:50.606 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57079.log 2026-02-20T21:28:50.606 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77267.log 2026-02-20T21:28:50.606 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.66534.log: /var/log/ceph/cluster2-client.admin.57079.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82847.log 2026-02-20T21:28:50.606 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.66534.log.gz 2026-02-20T21:28:50.606 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.77267.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.57079.log.gz 2026-02-20T21:28:50.607 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75243.log 2026-02-20T21:28:50.607 INFO:teuthology.orchestra.run.trial199.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.88829.log.gz 2026-02-20T21:28:50.607 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.82847.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.74450.log.gz 2026-02-20T21:28:50.607 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69741.log 2026-02-20T21:28:50.607 INFO:teuthology.orchestra.run.trial199.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.77267.log.gz 2026-02-20T21:28:50.607 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.82847.log.gz 2026-02-20T21:28:50.607 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48276.log 2026-02-20T21:28:50.607 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.75243.log: /var/log/ceph/cluster2-client.admin.69741.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74222.log 2026-02-20T21:28:50.607 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.75243.log.gz 2026-02-20T21:28:50.608 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.48276.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74029.log 2026-02-20T21:28:50.608 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.69741.log.gz 2026-02-20T21:28:50.608 INFO:teuthology.orchestra.run.trial199.stderr: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48276.log.gz 2026-02-20T21:28:50.608 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.74222.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84480.log 2026-02-20T21:28:50.608 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.74029.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73702.log 2026-02-20T21:28:50.608 INFO:teuthology.orchestra.run.trial199.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.74222.log.gz 2026-02-20T21:28:50.608 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.84480.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83593.log 2026-02-20T21:28:50.608 INFO:teuthology.orchestra.run.trial199.stderr: 90.3% 86.6% -- replaced with /var/log/ceph/cluster2-client.mirror.74029.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.84480.log.gz 2026-02-20T21:28:50.608 INFO:teuthology.orchestra.run.trial199.stderr: 2026-02-20T21:28:50.609 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.73702.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85724.log 2026-02-20T21:28:50.609 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.83593.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.73702.log.gz 2026-02-20T21:28:50.609 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75427.log 2026-02-20T21:28:50.609 INFO:teuthology.orchestra.run.trial199.stderr: 86.4% -- replaced with /var/log/ceph/cluster1-client.mirror.83593.log.gz 2026-02-20T21:28:50.609 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.85724.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83025.log 2026-02-20T21:28:50.609 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75427.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74262.log 2026-02-20T21:28:50.609 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.83025.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52641.log 2026-02-20T21:28:50.609 INFO:teuthology.orchestra.run.trial199.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.75427.log.gz 2026-02-20T21:28:50.610 INFO:teuthology.orchestra.run.trial199.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.83025.log.gz 2026-02-20T21:28:50.610 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.74262.log: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.85724.log.gz 2026-02-20T21:28:50.610 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72298.log 2026-02-20T21:28:50.610 INFO:teuthology.orchestra.run.trial199.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.74262.log.gz 2026-02-20T21:28:50.610 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62609.log 2026-02-20T21:28:50.610 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.52641.log: /var/log/ceph/cluster2-client.admin.72298.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67374.log 2026-02-20T21:28:50.610 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.52641.log.gz 2026-02-20T21:28:50.610 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.62609.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63448.log 2026-02-20T21:28:50.610 INFO:teuthology.orchestra.run.trial199.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.admin.72298.log.gz 2026-02-20T21:28:50.611 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.62609.log.gz 2026-02-20T21:28:50.611 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.67374.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53853.log 2026-02-20T21:28:50.611 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.67374.log.gz 2026-02-20T21:28:50.611 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.63448.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85300.log 2026-02-20T21:28:50.611 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.63448.log.gz 2026-02-20T21:28:50.611 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.53853.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74967.log 2026-02-20T21:28:50.611 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.53853.log.gz 2026-02-20T21:28:50.611 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.85300.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68050.log 2026-02-20T21:28:50.612 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.85300.log.gz 2026-02-20T21:28:50.612 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.74967.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74766.log 2026-02-20T21:28:50.612 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.68050.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86720.log 2026-02-20T21:28:50.612 INFO:teuthology.orchestra.run.trial199.stderr: 87.4%/var/log/ceph/cluster1-client.mirror.74766.log: -- replaced with /var/log/ceph/cluster1-client.admin.68050.log.gz 2026-02-20T21:28:50.612 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73377.log 2026-02-20T21:28:50.612 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.74766.log.gz 2026-02-20T21:28:50.612 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.86720.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53579.log 2026-02-20T21:28:50.612 INFO:teuthology.orchestra.run.trial199.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.74967.log.gz 2026-02-20T21:28:50.612 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.73377.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63748.log 2026-02-20T21:28:50.613 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.53579.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.86720.log.gz 2026-02-20T21:28:50.613 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76115.log 2026-02-20T21:28:50.613 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.73377.log.gz 2026-02-20T21:28:50.613 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53579.log.gz 2026-02-20T21:28:50.613 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.63748.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84573.log 2026-02-20T21:28:50.613 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76115.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53498.log 2026-02-20T21:28:50.613 INFO:teuthology.orchestra.run.trial199.stderr: 39.8% -- replaced with /var/log/ceph/cluster2-client.mirror.76115.log.gz 2026-02-20T21:28:50.613 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.84573.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83179.log 2026-02-20T21:28:50.613 INFO:teuthology.orchestra.run.trial199.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.63748.log.gz 2026-02-20T21:28:50.614 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.53498.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.84573.log.gz 2026-02-20T21:28:50.614 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75988.log 2026-02-20T21:28:50.614 INFO:teuthology.orchestra.run.trial199.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.53498.log.gz 2026-02-20T21:28:50.614 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.83179.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82633.log 2026-02-20T21:28:50.614 INFO:teuthology.orchestra.run.trial199.stderr: 64.6% -- replaced with /var/log/ceph/cluster1-client.mirror.83179.log.gz 2026-02-20T21:28:50.614 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75988.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79393.log 2026-02-20T21:28:50.614 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.82633.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54780.log 2026-02-20T21:28:50.614 INFO:teuthology.orchestra.run.trial199.stderr: 88.2% 69.4% -- replaced with /var/log/ceph/cluster2-client.mirror.75988.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.82633.log.gz 2026-02-20T21:28:50.614 INFO:teuthology.orchestra.run.trial199.stderr: 2026-02-20T21:28:50.614 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.79393.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64301.log 2026-02-20T21:28:50.614 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.54780.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.78494.log 2026-02-20T21:28:50.615 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.64301.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59160.log 2026-02-20T21:28:50.615 INFO:teuthology.orchestra.run.trial199.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.79393.log.gz 2026-02-20T21:28:50.615 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.78494.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.54780.log.gz 2026-02-20T21:28:50.615 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76915.log 2026-02-20T21:28:50.615 INFO:teuthology.orchestra.run.trial199.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.78494.log.gz 2026-02-20T21:28:50.615 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.64301.log.gz 2026-02-20T21:28:50.615 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51448.log 2026-02-20T21:28:50.615 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.59160.log: /var/log/ceph/cluster2-client.mirror.76915.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76599.log 2026-02-20T21:28:50.616 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.59160.log.gz 2026-02-20T21:28:50.616 INFO:teuthology.orchestra.run.trial199.stderr: 88.2%/var/log/ceph/cluster2-client.mirror.51448.log: -- replaced with /var/log/ceph/cluster2-client.mirror.76915.log.gz 2026-02-20T21:28:50.616 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47998.log 2026-02-20T21:28:50.616 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51448.log.gz 2026-02-20T21:28:50.616 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76599.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53891.log 2026-02-20T21:28:50.616 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.47998.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61132.log 2026-02-20T21:28:50.616 INFO:teuthology.orchestra.run.trial199.stderr: 64.3% -- replaced with /var/log/ceph/cluster1-client.mirror.47998.log.gz 2026-02-20T21:28:50.616 INFO:teuthology.orchestra.run.trial199.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.76599.log.gz 2026-02-20T21:28:50.616 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.53891.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76642.log 2026-02-20T21:28:50.617 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.61132.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57418.log 2026-02-20T21:28:50.617 INFO:teuthology.orchestra.run.trial199.stderr: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.1.77807.log.gz 2026-02-20T21:28:50.617 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.61132.log.gz 2026-02-20T21:28:50.617 INFO:teuthology.orchestra.run.trial199.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53891.log.gz 2026-02-20T21:28:50.617 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76642.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88691.log 2026-02-20T21:28:50.617 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.57418.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76642.log.gz 2026-02-20T21:28:50.617 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85323.log 2026-02-20T21:28:50.618 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.88691.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.57418.log.gz 2026-02-20T21:28:50.618 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65169.log 2026-02-20T21:28:50.618 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.88691.log.gz 2026-02-20T21:28:50.618 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.85323.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81834.log 2026-02-20T21:28:50.618 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.85323.log.gz 2026-02-20T21:28:50.618 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.65169.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56337.log 2026-02-20T21:28:50.618 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.81834.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.65169.log.gz 2026-02-20T21:28:50.618 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59344.log 2026-02-20T21:28:50.618 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.56337.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64711.log 2026-02-20T21:28:50.618 INFO:teuthology.orchestra.run.trial199.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.81834.log.gz 2026-02-20T21:28:50.619 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.59344.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66336.log 2026-02-20T21:28:50.619 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.64711.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60186.log 2026-02-20T21:28:50.619 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.59344.log.gz 2026-02-20T21:28:50.619 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.66336.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76400.log 2026-02-20T21:28:50.619 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% 90.5% -- replaced with /var/log/ceph/cluster1-client.admin.64711.log.gz 2026-02-20T21:28:50.619 INFO:teuthology.orchestra.run.trial199.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.56337.log.gz 2026-02-20T21:28:50.619 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.66336.log.gz 2026-02-20T21:28:50.619 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.60186.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59211.log 2026-02-20T21:28:50.620 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.76400.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.60186.log.gz 2026-02-20T21:28:50.620 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60132.log 2026-02-20T21:28:50.620 INFO:teuthology.orchestra.run.trial199.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.76400.log.gz 2026-02-20T21:28:50.620 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.59211.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88431.log 2026-02-20T21:28:50.620 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.60132.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.59211.log.gz 2026-02-20T21:28:50.620 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.90468.log 2026-02-20T21:28:50.620 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.60132.log.gz 2026-02-20T21:28:50.620 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.88431.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89662.log 2026-02-20T21:28:50.620 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.88431.log.gz 2026-02-20T21:28:50.620 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.90468.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.81119.log 2026-02-20T21:28:50.620 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.90468.log.gz 2026-02-20T21:28:50.620 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.89662.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.89389.log 2026-02-20T21:28:50.621 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.89662.log.gz 2026-02-20T21:28:50.621 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.1.81119.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82530.log 2026-02-20T21:28:50.621 INFO:teuthology.orchestra.run.trial199.stderr: 19.7% -- replaced with /var/log/ceph/cluster1-client.mirror.1.81119.log.gz 2026-02-20T21:28:50.621 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.89389.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64415.log 2026-02-20T21:28:50.621 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.82530.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.89389.log.gz 2026-02-20T21:28:50.621 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82895.log 2026-02-20T21:28:50.621 INFO:teuthology.orchestra.run.trial199.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.82530.log.gz 2026-02-20T21:28:50.621 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.64415.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50083.log 2026-02-20T21:28:50.622 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.82895.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.64415.log.gz 2026-02-20T21:28:50.622 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82305.log 2026-02-20T21:28:50.622 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.82895.log.gz 2026-02-20T21:28:50.622 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.50083.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69687.log 2026-02-20T21:28:50.622 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.82305.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.81172.log 2026-02-20T21:28:50.622 INFO:teuthology.orchestra.run.trial199.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50083.log.gz 2026-02-20T21:28:50.622 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.69687.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69862.log 2026-02-20T21:28:50.622 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.69687.log.gz 2026-02-20T21:28:50.622 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.2.81172.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64146.log 2026-02-20T21:28:50.623 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.69862.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82305.log.gz 2026-02-20T21:28:50.623 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53369.log 2026-02-20T21:28:50.623 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.69862.log.gz 2026-02-20T21:28:50.623 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.64146.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48225.log 2026-02-20T21:28:50.623 INFO:teuthology.orchestra.run.trial199.stderr: 87.7%/var/log/ceph/cluster1-client.admin.53369.log: -- replaced with /var/log/ceph/cluster2-client.mirror.64146.log.gz 2026-02-20T21:28:50.623 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74918.log 2026-02-20T21:28:50.623 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.48225.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.53369.log.gz 2026-02-20T21:28:50.623 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.91097.log 2026-02-20T21:28:50.623 INFO:teuthology.orchestra.run.trial199.stderr: 70.9% -- replaced with /var/log/ceph/cluster2-client.admin.48225.log.gz 2026-02-20T21:28:50.623 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.74918.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53982.log 2026-02-20T21:28:50.624 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.91097.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72594.log 2026-02-20T21:28:50.624 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.91097.log.gz 2026-02-20T21:28:50.624 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.53982.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64738.log 2026-02-20T21:28:50.624 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.72594.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64388.log 2026-02-20T21:28:50.624 INFO:teuthology.orchestra.run.trial199.stderr: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.74918.log.gz 2026-02-20T21:28:50.624 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.64738.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53982.log.gz 2026-02-20T21:28:50.624 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58709.log 2026-02-20T21:28:50.624 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.64738.log.gz 2026-02-20T21:28:50.625 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.89759.log 2026-02-20T21:28:50.625 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.64388.log: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.2.81172.log.gz 2026-02-20T21:28:50.625 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.58709.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79172.log 2026-02-20T21:28:50.625 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.64388.log.gz 2026-02-20T21:28:50.625 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.2.89759.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.72594.log.gz 2026-02-20T21:28:50.625 INFO:teuthology.orchestra.run.trial199.stderr: 93.4% 19.7% -- replaced with /var/log/ceph/cluster2-client.mirror.2.54234.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.2.89759.log.gz 2026-02-20T21:28:50.625 INFO:teuthology.orchestra.run.trial199.stderr: 2026-02-20T21:28:50.625 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75673.log 2026-02-20T21:28:50.625 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.79172.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77061.log 2026-02-20T21:28:50.625 INFO:teuthology.orchestra.run.trial199.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58709.log.gz 2026-02-20T21:28:50.625 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75673.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60708.log 2026-02-20T21:28:50.626 INFO:teuthology.orchestra.run.trial199.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.79172.log.gz 2026-02-20T21:28:50.626 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.77061.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58464.log 2026-02-20T21:28:50.626 INFO:teuthology.orchestra.run.trial199.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.75673.log.gz 87.7% 2026-02-20T21:28:50.626 INFO:teuthology.orchestra.run.trial199.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.77061.log.gz 2026-02-20T21:28:50.626 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.60708.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74996.log 2026-02-20T21:28:50.626 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.mirror.60708.log.gz 2026-02-20T21:28:50.626 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.58464.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83830.log 2026-02-20T21:28:50.626 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.58464.log.gz 2026-02-20T21:28:50.626 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.74996.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75928.log 2026-02-20T21:28:50.627 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.83830.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49119.log 2026-02-20T21:28:50.627 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75928.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61886.log 2026-02-20T21:28:50.627 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.49119.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.83830.log.gz 2026-02-20T21:28:50.627 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65969.log 2026-02-20T21:28:50.627 INFO:teuthology.orchestra.run.trial199.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75928.log.gz 2026-02-20T21:28:50.627 INFO:teuthology.orchestra.run.trial199.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74996.log.gz 2026-02-20T21:28:50.627 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.61886.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50291.log 2026-02-20T21:28:50.627 INFO:teuthology.orchestra.run.trial199.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49119.log.gz 2026-02-20T21:28:50.627 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.65969.log: gzip 87.6% -5 -- replaced with /var/log/ceph/cluster2-client.admin.61886.log.gz --verbose 2026-02-20T21:28:50.627 INFO:teuthology.orchestra.run.trial199.stderr: -- /var/log/ceph/cluster2-client.mirror.53542.log 2026-02-20T21:28:50.628 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.50291.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49684.log 2026-02-20T21:28:50.628 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.50291.log.gz 2026-02-20T21:28:50.628 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.53542.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77408.log 2026-02-20T21:28:50.628 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.65969.log.gz 2026-02-20T21:28:50.628 INFO:teuthology.orchestra.run.trial199.stderr: 66.1% -- replaced with /var/log/ceph/cluster2-client.mirror.53542.log.gz 2026-02-20T21:28:50.628 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.49684.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68960.log 2026-02-20T21:28:50.628 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.77408.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.49684.log.gz 2026-02-20T21:28:50.628 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55749.log 2026-02-20T21:28:50.629 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.77408.log.gz 2026-02-20T21:28:50.629 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.68960.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67922.log 2026-02-20T21:28:50.629 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.55749.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.68960.log.gz 2026-02-20T21:28:50.629 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84012.log 2026-02-20T21:28:50.629 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.55749.log.gz 2026-02-20T21:28:50.629 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.67922.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73596.log 2026-02-20T21:28:50.629 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.84012.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57865.log 2026-02-20T21:28:50.629 INFO:teuthology.orchestra.run.trial199.stderr: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.0.50691.log.gz 2026-02-20T21:28:50.629 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.73596.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56411.log 2026-02-20T21:28:50.629 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.57865.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58735.log 2026-02-20T21:28:50.630 INFO:teuthology.orchestra.run.trial199.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.67922.log.gz 2026-02-20T21:28:50.630 INFO:teuthology.orchestra.run.trial199.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.73596.log.gz 2026-02-20T21:28:50.630 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.57865.log.gz 2026-02-20T21:28:50.630 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.56411.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83090.log 2026-02-20T21:28:50.630 INFO:teuthology.orchestra.run.trial199.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.84012.log.gz 2026-02-20T21:28:50.630 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.56411.log.gz 2026-02-20T21:28:50.630 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.58735.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56204.log 2026-02-20T21:28:50.630 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.58735.log.gz 2026-02-20T21:28:50.630 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.83090.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55043.log 2026-02-20T21:28:50.630 INFO:teuthology.orchestra.run.trial199.stderr: 86.5% -- replaced with /var/log/ceph/cluster1-client.mirror.83090.log.gz 2026-02-20T21:28:50.631 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.56204.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77287.log 2026-02-20T21:28:50.631 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.55043.log: 87.7%gzip -- replaced with /var/log/ceph/cluster1-client.admin.56204.log.gz 2026-02-20T21:28:50.631 INFO:teuthology.orchestra.run.trial199.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87847.log 2026-02-20T21:28:50.631 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.77287.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.55043.log.gz 2026-02-20T21:28:50.631 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46585.log 2026-02-20T21:28:50.631 INFO:teuthology.orchestra.run.trial199.stderr: 87.8%/var/log/ceph/cluster2-client.mirror.87847.log: -- replaced with /var/log/ceph/cluster2-client.mirror.77287.log.gz 2026-02-20T21:28:50.631 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71519.log 2026-02-20T21:28:50.631 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.87847.log.gz 2026-02-20T21:28:50.631 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.46585.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62797.log 2026-02-20T21:28:50.631 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46585.log.gz 2026-02-20T21:28:50.631 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.71519.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66789.log 2026-02-20T21:28:50.632 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.62797.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74947.log 2026-02-20T21:28:50.632 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.66789.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.62797.log.gz 2026-02-20T21:28:50.632 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55420.log 2026-02-20T21:28:50.632 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.71519.log.gz 2026-02-20T21:28:50.632 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.66789.log.gz 2026-02-20T21:28:50.632 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.74947.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81237.log 2026-02-20T21:28:50.632 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.55420.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63347.log 2026-02-20T21:28:50.632 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.81237.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74475.log 2026-02-20T21:28:50.633 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.63347.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.81237.log.gz 2026-02-20T21:28:50.633 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.80433.log 2026-02-20T21:28:50.633 INFO:teuthology.orchestra.run.trial199.stderr: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.74947.log.gz 2026-02-20T21:28:50.633 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.74475.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.55420.log.gz 2026-02-20T21:28:50.633 INFO:teuthology.orchestra.run.trial199.stderr: 80.4% -- replaced with /var/log/ceph/cluster2-client.mirror.74475.log.gz 2026-02-20T21:28:50.633 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65791.log 2026-02-20T21:28:50.633 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72942.log 2026-02-20T21:28:50.633 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.80433.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63347.log.gz 2026-02-20T21:28:50.633 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.65791.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78804.log 2026-02-20T21:28:50.633 INFO:teuthology.orchestra.run.trial199.stderr: 90.2%/var/log/ceph/cluster1-client.mirror.72942.log: -- replaced with /var/log/ceph/cluster2-client.admin.80433.log.gz 2026-02-20T21:28:50.633 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.65791.log.gz 2026-02-20T21:28:50.634 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.77763.log 2026-02-20T21:28:50.634 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.72942.log.gz 2026-02-20T21:28:50.634 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.78804.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68424.log 2026-02-20T21:28:50.634 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.78804.log.gz 2026-02-20T21:28:50.634 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.0.77763.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56892.log 2026-02-20T21:28:50.634 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.68424.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69230.log 2026-02-20T21:28:50.634 INFO:teuthology.orchestra.run.trial199.stderr: 93.9% -- replaced with /var/log/ceph/cluster1-client.mirror.88849.log.gz 2026-02-20T21:28:50.634 INFO:teuthology.orchestra.run.trial199.stderr: 87.5%/var/log/ceph/cluster1-client.admin.56892.log: -- replaced with /var/log/ceph/cluster1-client.admin.68424.log.gz 2026-02-20T21:28:50.634 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70235.log 2026-02-20T21:28:50.634 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.69230.log: 87.5%gzip -- replaced with /var/log/ceph/cluster1-client.admin.56892.log.gz -5 2026-02-20T21:28:50.635 INFO:teuthology.orchestra.run.trial199.stderr: --verbose -- /var/log/ceph/cluster1-client.admin.68906.log 2026-02-20T21:28:50.635 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.70235.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62038.log 2026-02-20T21:28:50.635 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.70235.log.gz 2026-02-20T21:28:50.635 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.68906.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51953.log 2026-02-20T21:28:50.635 INFO:teuthology.orchestra.run.trial199.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.69230.log.gz 2026-02-20T21:28:50.635 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.68906.log.gz 2026-02-20T21:28:50.635 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.62038.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66218.log 2026-02-20T21:28:50.635 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.62038.log.gz 2026-02-20T21:28:50.636 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.51953.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53046.log 2026-02-20T21:28:50.636 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.66218.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.51953.log.gz 2026-02-20T21:28:50.636 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84295.log 2026-02-20T21:28:50.636 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.53046.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49955.log 2026-02-20T21:28:50.636 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.84295.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55608.log 2026-02-20T21:28:50.636 INFO:teuthology.orchestra.run.trial199.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.53046.log.gz 2026-02-20T21:28:50.636 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.66218.log.gz 2026-02-20T21:28:50.636 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.84295.log.gz 2026-02-20T21:28:50.636 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.49955.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72510.log 2026-02-20T21:28:50.636 INFO:teuthology.orchestra.run.trial199.stderr: 68.8% -- replaced with /var/log/ceph/cluster1-client.mirror.49955.log.gz 2026-02-20T21:28:50.637 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.55608.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75694.log 2026-02-20T21:28:50.637 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.72510.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79145.log 2026-02-20T21:28:50.637 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75694.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65195.log 2026-02-20T21:28:50.637 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.79145.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60437.log 2026-02-20T21:28:50.637 INFO:teuthology.orchestra.run.trial199.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.75694.log.gz 2026-02-20T21:28:50.637 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% 91.8% -- replaced with /var/log/ceph/cluster1-client.admin.79145.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.55608.log.gz 2026-02-20T21:28:50.637 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.65195.log: 2026-02-20T21:28:50.637 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78852.log 2026-02-20T21:28:50.637 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.60437.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58759.log 2026-02-20T21:28:50.638 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.78852.log: gzip -5 89.7% --verbose -- replaced with /var/log/ceph/cluster1-client.mirror.65195.log.gz -- 2026-02-20T21:28:50.638 INFO:teuthology.orchestra.run.trial199.stderr: /var/log/ceph/cluster2-client.admin.43429.log 2026-02-20T21:28:50.638 INFO:teuthology.orchestra.run.trial199.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.mirror.0.77763.log.gz 2026-02-20T21:28:50.638 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.78852.log.gz 2026-02-20T21:28:50.638 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.58759.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60437.log.gz 2026-02-20T21:28:50.638 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49502.log 2026-02-20T21:28:50.638 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.58759.log.gz 2026-02-20T21:28:50.638 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69390.log 2026-02-20T21:28:50.638 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.43429.log: 28.5% -- replaced with /var/log/ceph/cluster2-client.admin.43429.log.gz 2026-02-20T21:28:50.638 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.49502.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.86262.log 2026-02-20T21:28:50.638 INFO:teuthology.orchestra.run.trial199.stderr: 80.2% -- replaced with /var/log/ceph/cluster2-client.mirror.49502.log.gz 2026-02-20T21:28:50.639 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.69390.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69460.log 2026-02-20T21:28:50.639 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.3.86262.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55070.log 2026-02-20T21:28:50.639 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.69460.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.45949.log 2026-02-20T21:28:50.639 INFO:teuthology.orchestra.run.trial199.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.69390.log.gz 2026-02-20T21:28:50.639 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.55070.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71162.log 2026-02-20T21:28:50.639 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.69460.log.gz 2026-02-20T21:28:50.639 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.45949.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.55070.log.gz 2026-02-20T21:28:50.639 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56841.log 2026-02-20T21:28:50.639 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.45949.log.gz 2026-02-20T21:28:50.640 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.71162.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70636.log 2026-02-20T21:28:50.640 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.56841.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.48543.log 2026-02-20T21:28:50.640 INFO:teuthology.orchestra.run.trial199.stderr: 87.6%/var/log/ceph/cluster1-client.mirror.70636.log: -- replaced with /var/log/ceph/cluster2-client.mirror.56841.log.gz 2026-02-20T21:28:50.640 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66507.log 2026-02-20T21:28:50.640 INFO:teuthology.orchestra.run.trial199.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.71162.log.gz 2026-02-20T21:28:50.640 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.70636.log.gz 2026-02-20T21:28:50.640 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.2.48543.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72699.log 2026-02-20T21:28:50.640 INFO:teuthology.orchestra.run.trial199.stderr: 19.1% -- replaced with /var/log/ceph/cluster1-client.mirror.2.48543.log.gz 2026-02-20T21:28:50.640 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.66507.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81065.log 2026-02-20T21:28:50.640 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.72699.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.66507.log.gz 2026-02-20T21:28:50.641 INFO:teuthology.orchestra.run.trial199.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.72510.log.gz 2026-02-20T21:28:50.641 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.88715.log 2026-02-20T21:28:50.641 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.81065.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79049.log 2026-02-20T21:28:50.641 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72699.log.gz 2026-02-20T21:28:50.641 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.88715.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79943.log 2026-02-20T21:28:50.641 INFO:teuthology.orchestra.run.trial199.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81065.log.gz 2026-02-20T21:28:50.641 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.88715.log.gz 2026-02-20T21:28:50.641 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.79049.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83654.log 2026-02-20T21:28:50.641 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.79943.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68504.log 2026-02-20T21:28:50.641 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.79049.log.gz 2026-02-20T21:28:50.642 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.83654.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58362.log 2026-02-20T21:28:50.642 INFO:teuthology.orchestra.run.trial199.stderr: 56.7% -- replaced with /var/log/ceph/cluster2-client.mirror.83654.log.gz 2026-02-20T21:28:50.642 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.68504.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53215.log 2026-02-20T21:28:50.642 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.58362.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81689.log 2026-02-20T21:28:50.642 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.53215.log: 94.3% 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.3.86262.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.58362.log.gz 2026-02-20T21:28:50.642 INFO:teuthology.orchestra.run.trial199.stderr: 2026-02-20T21:28:50.642 INFO:teuthology.orchestra.run.trial199.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.68504.log.gz 2026-02-20T21:28:50.642 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49169.log 2026-02-20T21:28:50.642 INFO:teuthology.orchestra.run.trial199.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.79943.log.gz 2026-02-20T21:28:50.643 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.81689.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69177.log 2026-02-20T21:28:50.643 INFO:teuthology.orchestra.run.trial199.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.81689.log.gz 2026-02-20T21:28:50.643 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.49169.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60892.log 2026-02-20T21:28:50.643 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.49169.log.gz 2026-02-20T21:28:50.643 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.69177.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.53215.log.gz 2026-02-20T21:28:50.643 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.54650.log 2026-02-20T21:28:50.643 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.69177.log.gz 2026-02-20T21:28:50.643 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73943.log 2026-02-20T21:28:50.643 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.60892.log: /var/log/ceph/cluster2-client.admin.54650.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54968.log 2026-02-20T21:28:50.644 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.60892.log.gz 2026-02-20T21:28:50.644 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.73943.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82069.log 87.4% 2026-02-20T21:28:50.644 INFO:teuthology.orchestra.run.trial199.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.54650.log.gz 2026-02-20T21:28:50.644 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.54968.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.86081.log 2026-02-20T21:28:50.644 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.82069.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51307.log 2026-02-20T21:28:50.644 INFO:teuthology.orchestra.run.trial199.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73943.log.gz 2026-02-20T21:28:50.644 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.1.86081.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56102.log 2026-02-20T21:28:50.644 INFO:teuthology.orchestra.run.trial199.stderr: 19.1% -- replaced with /var/log/ceph/cluster2-client.mirror.1.86081.log.gz 2026-02-20T21:28:50.644 INFO:teuthology.orchestra.run.trial199.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.82069.log.gz 2026-02-20T21:28:50.645 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.51307.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63825.log 2026-02-20T21:28:50.645 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.56102.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49938.log 2026-02-20T21:28:50.645 INFO:teuthology.orchestra.run.trial199.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.54968.log.gz 2026-02-20T21:28:50.645 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.admin.51307.log.gz 2026-02-20T21:28:50.645 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.63825.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85504.log 2026-02-20T21:28:50.645 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.49938.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.63825.log.gz 2026-02-20T21:28:50.645 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81531.log 2026-02-20T21:28:50.645 INFO:teuthology.orchestra.run.trial199.stderr: 57.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49938.log.gz 2026-02-20T21:28:50.645 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.85504.log: 80.5% -- replaced with /var/log/ceph/cluster2-client.mirror.85504.log.gz 2026-02-20T21:28:50.646 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85147.log 2026-02-20T21:28:50.646 INFO:teuthology.orchestra.run.trial199.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.mirror.56102.log.gz 2026-02-20T21:28:50.646 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79436.log 2026-02-20T21:28:50.646 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.81531.log: /var/log/ceph/cluster1-client.mirror.85147.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57025.log 2026-02-20T21:28:50.646 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.81531.log.gz 2026-02-20T21:28:50.646 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.79436.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79286.log 2026-02-20T21:28:50.646 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.57025.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49812.log 2026-02-20T21:28:50.646 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.57025.log.gz 2026-02-20T21:28:50.646 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.79286.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47068.log 2026-02-20T21:28:50.646 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.49812.log: 87.4%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.85147.log.gz -5 2026-02-20T21:28:50.647 INFO:teuthology.orchestra.run.trial199.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.2.86185.log 2026-02-20T21:28:50.647 INFO:teuthology.orchestra.run.trial199.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.admin.79286.log.gz 2026-02-20T21:28:50.647 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.47068.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68847.log 2026-02-20T21:28:50.647 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47068.log.gz 2026-02-20T21:28:50.647 INFO:teuthology.orchestra.run.trial199.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.79436.log.gz 2026-02-20T21:28:50.647 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.2.86185.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46676.log 2026-02-20T21:28:50.647 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.68847.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48310.log 2026-02-20T21:28:50.647 INFO:teuthology.orchestra.run.trial199.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.49812.log.gz 2026-02-20T21:28:50.648 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.46676.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74898.log 2026-02-20T21:28:50.648 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.68847.log.gz 2026-02-20T21:28:50.648 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46676.log.gz 2026-02-20T21:28:50.648 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.48310.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86972.log 2026-02-20T21:28:50.648 INFO:teuthology.orchestra.run.trial199.stderr: 64.3% -- replaced with /var/log/ceph/cluster2-client.mirror.48310.log.gz 2026-02-20T21:28:50.648 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.74898.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79460.log 2026-02-20T21:28:50.648 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.86972.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75059.log 2026-02-20T21:28:50.648 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.79460.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.86972.log.gz 2026-02-20T21:28:50.648 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56465.log 2026-02-20T21:28:50.648 INFO:teuthology.orchestra.run.trial199.stderr: 80.3% -- replaced with /var/log/ceph/cluster2-client.mirror.79460.log.gz 2026-02-20T21:28:50.649 INFO:teuthology.orchestra.run.trial199.stderr:gzip/var/log/ceph/cluster2-client.mirror.75059.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83128.log 2026-02-20T21:28:50.649 INFO:teuthology.orchestra.run.trial199.stderr: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74898.log.gz 2026-02-20T21:28:50.649 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.56465.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77123.log 2026-02-20T21:28:50.649 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.83128.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48393.log 2026-02-20T21:28:50.649 INFO:teuthology.orchestra.run.trial199.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.56465.log.gz 2026-02-20T21:28:50.649 INFO:teuthology.orchestra.run.trial199.stderr: 50.6% -- replaced with /var/log/ceph/cluster2-client.mirror.83128.log.gz 2026-02-20T21:28:50.649 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.77123.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76050.log 2026-02-20T21:28:50.649 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.48393.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.77585.log 2026-02-20T21:28:50.649 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48393.log.gz 2026-02-20T21:28:50.649 INFO:teuthology.orchestra.run.trial199.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.77123.log.gz 2026-02-20T21:28:50.649 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.76050.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71599.log 2026-02-20T21:28:50.650 INFO:teuthology.orchestra.run.trial199.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.75059.log.gz 2026-02-20T21:28:50.650 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.76050.log.gz 2026-02-20T21:28:50.650 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.77585.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86277.log 2026-02-20T21:28:50.650 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.71599.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59738.log 2026-02-20T21:28:50.650 INFO:teuthology.orchestra.run.trial199.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.admin.77585.log.gz 2026-02-20T21:28:50.650 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.71599.log.gz 2026-02-20T21:28:50.650 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.86277.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.90289.log 2026-02-20T21:28:50.650 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.86277.log.gz 2026-02-20T21:28:50.650 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.59738.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59944.log 2026-02-20T21:28:50.651 INFO:teuthology.orchestra.run.trial199.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.2.86185.log.gz 2026-02-20T21:28:50.651 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.59738.log.gz 2026-02-20T21:28:50.651 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.90289.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70097.log 2026-02-20T21:28:50.651 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.59944.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59452.log 2026-02-20T21:28:50.651 INFO:teuthology.orchestra.run.trial199.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.90289.log.gz 2026-02-20T21:28:50.651 INFO:teuthology.orchestra.run.trial199.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.59944.log.gz 2026-02-20T21:28:50.651 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.70097.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65686.log 2026-02-20T21:28:50.651 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.70097.log.gz 2026-02-20T21:28:50.651 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86551.log 2026-02-20T21:28:50.652 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.59452.log: /var/log/ceph/cluster1-client.mirror.65686.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48047.log 2026-02-20T21:28:50.652 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.65686.log.gz 2026-02-20T21:28:50.652 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.86551.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48327.log 2026-02-20T21:28:50.652 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.86551.log.gz 2026-02-20T21:28:50.652 INFO:teuthology.orchestra.run.trial199.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.59452.log.gz 2026-02-20T21:28:50.652 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.48047.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78778.log 2026-02-20T21:28:50.652 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48047.log.gz 2026-02-20T21:28:50.652 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.48327.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.42869.log 2026-02-20T21:28:50.652 INFO:teuthology.orchestra.run.trial199.stderr: 64.1% -- replaced with /var/log/ceph/cluster2-client.mirror.48327.log.gz 2026-02-20T21:28:50.653 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.78778.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65088.log 2026-02-20T21:28:50.653 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.42869.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71338.log 2026-02-20T21:28:50.653 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.42869.log.gz 2026-02-20T21:28:50.653 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.65088.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76894.log 2026-02-20T21:28:50.653 INFO:teuthology.orchestra.run.trial199.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.78778.log.gz 2026-02-20T21:28:50.653 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.71338.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.65088.log.gz 2026-02-20T21:28:50.653 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48191.log 2026-02-20T21:28:50.653 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76894.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53455.log 2026-02-20T21:28:50.653 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88479.log 2026-02-20T21:28:50.653 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.48191.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.76894.log.gz 2026-02-20T21:28:50.653 INFO:teuthology.orchestra.run.trial199.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.71338.log.gz 2026-02-20T21:28:50.653 INFO:teuthology.orchestra.run.trial199.stderr: 80.1% -- replaced with /var/log/ceph/cluster2-client.admin.48191.log.gz 2026-02-20T21:28:50.653 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.53455.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47166.log 2026-02-20T21:28:50.654 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.53455.log.gz 2026-02-20T21:28:50.654 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.88479.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.54854.log 2026-02-20T21:28:50.654 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.88479.log.gz 2026-02-20T21:28:50.654 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.47166.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47166.log.gz 2026-02-20T21:28:50.654 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.77865.log 2026-02-20T21:28:50.654 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.54854.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83528.log 2026-02-20T21:28:50.654 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.3.77865.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64888.log 2026-02-20T21:28:50.654 INFO:teuthology.orchestra.run.trial199.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.54854.log.gz 2026-02-20T21:28:50.654 INFO:teuthology.orchestra.run.trial199.stderr: 18.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.77865.log.gz 2026-02-20T21:28:50.655 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.83528.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66480.log 2026-02-20T21:28:50.655 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.64888.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53828.log 2026-02-20T21:28:50.655 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.66480.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60509.log 2026-02-20T21:28:50.655 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.64888.log.gz 2026-02-20T21:28:50.655 INFO:teuthology.orchestra.run.trial199.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.66480.log.gz 2026-02-20T21:28:50.655 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50491.log 2026-02-20T21:28:50.655 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.53828.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.53828.log.gz 2026-02-20T21:28:50.656 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.60509.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.83528.log.gz 2026-02-20T21:28:50.656 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53252.log 2026-02-20T21:28:50.656 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.50491.log: gzip 87.6% -5 -- replaced with /var/log/ceph/cluster1-client.admin.60509.log.gz --verbose 2026-02-20T21:28:50.656 INFO:teuthology.orchestra.run.trial199.stderr: -- /var/log/ceph/cluster2-client.mirror.86740.log 2026-02-20T21:28:50.656 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.53252.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.50491.log.gz 2026-02-20T21:28:50.656 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60306.log 2026-02-20T21:28:50.656 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.53252.log.gz 2026-02-20T21:28:50.656 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.86740.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56284.log 2026-02-20T21:28:50.656 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.86740.log.gz 2026-02-20T21:28:50.656 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.60306.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.48542.log 2026-02-20T21:28:50.657 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.56284.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.60306.log.gz 2026-02-20T21:28:50.657 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52286.log 2026-02-20T21:28:50.657 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.1.48542.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83045.log 2026-02-20T21:28:50.657 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.52286.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58036.log 2026-02-20T21:28:50.657 INFO:teuthology.orchestra.run.trial199.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.56284.log.gz 2026-02-20T21:28:50.657 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.83045.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.admin.52286.log.gz 2026-02-20T21:28:50.657 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64170.log 2026-02-20T21:28:50.657 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.58036.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75848.log 2026-02-20T21:28:50.657 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.64170.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.48498.log 2026-02-20T21:28:50.657 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.58036.log.gz 2026-02-20T21:28:50.658 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.64170.log.gz 2026-02-20T21:28:50.658 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.75848.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60212.log 2026-02-20T21:28:50.658 INFO:teuthology.orchestra.run.trial199.stderr: 80.5% -- replaced with /var/log/ceph/cluster2-client.mirror.75848.log.gz 2026-02-20T21:28:50.658 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.0.48498.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82022.log 2026-02-20T21:28:50.658 INFO:teuthology.orchestra.run.trial199.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.83045.log.gz 2026-02-20T21:28:50.658 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.60212.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84165.log 2026-02-20T21:28:50.658 INFO:teuthology.orchestra.run.trial199.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.1.48542.log.gz 2026-02-20T21:28:50.658 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.82022.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60159.log 2026-02-20T21:28:50.658 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.82022.log.gz 2026-02-20T21:28:50.659 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.84165.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.60212.log.gz 2026-02-20T21:28:50.659 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71863.log 2026-02-20T21:28:50.659 INFO:teuthology.orchestra.run.trial199.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.84165.log.gz 2026-02-20T21:28:50.659 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.60159.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61299.log 2026-02-20T21:28:50.659 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.60159.log.gz 2026-02-20T21:28:50.659 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90342.log 2026-02-20T21:28:50.659 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.71863.log: /var/log/ceph/cluster2-client.admin.61299.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69204.log 2026-02-20T21:28:50.659 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.71863.log.gz 2026-02-20T21:28:50.659 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.90342.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.61299.log.gz 2026-02-20T21:28:50.659 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58491.log 2026-02-20T21:28:50.660 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55776.log 2026-02-20T21:28:50.660 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.69204.log: /var/log/ceph/cluster2-client.admin.58491.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.69204.log.gz 2026-02-20T21:28:50.660 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.78351.log 2026-02-20T21:28:50.660 INFO:teuthology.orchestra.run.trial199.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.87052.log.gz 2026-02-20T21:28:50.660 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.58491.log.gz 2026-02-20T21:28:50.660 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.55776.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68766.log 2026-02-20T21:28:50.661 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.55776.log.gz 2026-02-20T21:28:50.661 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.78351.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56438.log 2026-02-20T21:28:50.661 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.78351.log.gz 2026-02-20T21:28:50.661 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.68766.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79690.log 2026-02-20T21:28:50.661 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83880.log 2026-02-20T21:28:50.661 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.56438.log: /var/log/ceph/cluster1-client.mirror.79690.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72658.log 2026-02-20T21:28:50.661 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.56438.log.gz 2026-02-20T21:28:50.661 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.79690.log.gz 2026-02-20T21:28:50.661 INFO:teuthology.orchestra.run.trial199.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.68766.log.gz 2026-02-20T21:28:50.661 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.83880.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72739.log 2026-02-20T21:28:50.662 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.83880.log.gz 2026-02-20T21:28:50.662 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.72658.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50006.log 2026-02-20T21:28:50.662 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72658.log.gz 2026-02-20T21:28:50.662 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.72739.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63680.log 2026-02-20T21:28:50.662 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.50006.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46288.log 2026-02-20T21:28:50.662 INFO:teuthology.orchestra.run.trial199.stderr: 64.3% -- replaced with /var/log/ceph/cluster2-client.mirror.50006.log.gz 2026-02-20T21:28:50.662 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.72739.log.gz 2026-02-20T21:28:50.662 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.63680.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71809.log 2026-02-20T21:28:50.662 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.46288.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50026.log 2026-02-20T21:28:50.663 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46288.log.gz 2026-02-20T21:28:50.663 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.71809.log: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.63680.log.gz 2026-02-20T21:28:50.663 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.54191.log 2026-02-20T21:28:50.663 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.50026.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58437.log 2026-02-20T21:28:50.663 INFO:teuthology.orchestra.run.trial199.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.71809.log.gz 2026-02-20T21:28:50.663 INFO:teuthology.orchestra.run.trial199.stderr: 80.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50026.log.gz 2026-02-20T21:28:50.663 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.2.54191.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56525.log 2026-02-20T21:28:50.663 INFO:teuthology.orchestra.run.trial199.stderr: 18.6% -- replaced with /var/log/ceph/cluster2-client.mirror.2.54191.log.gz 2026-02-20T21:28:50.664 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.58437.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50342.log 2026-02-20T21:28:50.664 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.56525.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.58437.log.gzgzip 2026-02-20T21:28:50.664 INFO:teuthology.orchestra.run.trial199.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.admin.85357.log 2026-02-20T21:28:50.664 INFO:teuthology.orchestra.run.trial199.stderr: 93.8% -- replaced with /var/log/ceph/cluster1-client.mirror.90342.log.gz 2026-02-20T21:28:50.664 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.50342.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70151.log 2026-02-20T21:28:50.664 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.85357.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50603.log 2026-02-20T21:28:50.664 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.50342.log.gz 2026-02-20T21:28:50.664 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.70151.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77226.log 2026-02-20T21:28:50.664 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.85357.log.gz 2026-02-20T21:28:50.664 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.50603.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.78575.log 2026-02-20T21:28:50.664 INFO:teuthology.orchestra.run.trial199.stderr: 91.3% 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56525.log.gz -- replaced with /var/log/ceph/cluster2-client.admin.70151.log.gz 2026-02-20T21:28:50.664 INFO:teuthology.orchestra.run.trial199.stderr: 2026-02-20T21:28:50.664 INFO:teuthology.orchestra.run.trial199.stderr: 80.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50603.log.gz 2026-02-20T21:28:50.665 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.77226.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79985.log 2026-02-20T21:28:50.665 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.78575.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49972.log 2026-02-20T21:28:50.665 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.79985.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85246.log 2026-02-20T21:28:50.665 INFO:teuthology.orchestra.run.trial199.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.admin.78575.log.gz 2026-02-20T21:28:50.665 INFO:teuthology.orchestra.run.trial199.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79985.log.gz 2026-02-20T21:28:50.665 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55548.log 2026-02-20T21:28:50.665 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.49972.log: 70.0% -- replaced with /var/log/ceph/cluster2-client.mirror.49972.log.gz 2026-02-20T21:28:50.665 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.85246.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56022.log 2026-02-20T21:28:50.665 INFO:teuthology.orchestra.run.trial199.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.77226.log.gz 2026-02-20T21:28:50.665 INFO:teuthology.orchestra.run.trial199.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.85246.log.gz 2026-02-20T21:28:50.666 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.55548.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.48680.log 2026-02-20T21:28:50.666 INFO:teuthology.orchestra.run.trial199.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.55548.log.gz 2026-02-20T21:28:50.666 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67174.log 2026-02-20T21:28:50.666 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.56022.log: /var/log/ceph/cluster1-client.mirror.3.48680.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63600.log 2026-02-20T21:28:50.666 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.56022.log.gz 2026-02-20T21:28:50.666 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.67174.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57106.log 2026-02-20T21:28:50.666 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67174.log.gz 2026-02-20T21:28:50.666 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.63600.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56919.log 2026-02-20T21:28:50.667 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85972.log 2026-02-20T21:28:50.667 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.57106.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.63600.log.gz 2026-02-20T21:28:50.667 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.56919.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.54170.log 2026-02-20T21:28:50.667 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.57106.log.gz 2026-02-20T21:28:50.667 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.85972.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.56919.log.gz 2026-02-20T21:28:50.667 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73070.log 2026-02-20T21:28:50.667 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.0.54170.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.85972.log.gz 2026-02-20T21:28:50.667 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83508.log 2026-02-20T21:28:50.667 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.73070.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79714.log 2026-02-20T21:28:50.668 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.83508.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63421.log 2026-02-20T21:28:50.668 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.79714.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86760.log 2026-02-20T21:28:50.668 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.63421.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.admin.79714.log.gz 2026-02-20T21:28:50.668 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64792.log 2026-02-20T21:28:50.668 INFO:teuthology.orchestra.run.trial199.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.73070.log.gz 2026-02-20T21:28:50.668 INFO:teuthology.orchestra.run.trial199.stderr: 88.6%/var/log/ceph/cluster2-client.mirror.86760.log: -- replaced with /var/log/ceph/cluster2-client.admin.63421.log.gz 2026-02-20T21:28:50.668 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61245.log 2026-02-20T21:28:50.668 INFO:teuthology.orchestra.run.trial199.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.3.48680.log.gz 2026-02-20T21:28:50.669 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.64792.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51881.log 2026-02-20T21:28:50.669 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.64792.log.gz 2026-02-20T21:28:50.669 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.61245.log: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.83508.log.gz 2026-02-20T21:28:50.669 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62314.log 2026-02-20T21:28:50.669 INFO:teuthology.orchestra.run.trial199.stderr: 87.4%/var/log/ceph/cluster1-client.admin.51881.log: -- replaced with /var/log/ceph/cluster1-client.admin.61245.log.gz 2026-02-20T21:28:50.669 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66037.log 2026-02-20T21:28:50.669 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.62314.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.51881.log.gz 2026-02-20T21:28:50.669 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84121.log 2026-02-20T21:28:50.669 INFO:teuthology.orchestra.run.trial199.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.62314.log.gz 2026-02-20T21:28:50.669 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.91488.log 2026-02-20T21:28:50.669 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.66037.log: /var/log/ceph/cluster1-client.mirror.84121.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63262.log 2026-02-20T21:28:50.670 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.91488.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.84121.log.gz 2026-02-20T21:28:50.670 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81998.log 2026-02-20T21:28:50.670 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.91488.log.gz 2026-02-20T21:28:50.670 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.63262.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.66037.log.gz 2026-02-20T21:28:50.670 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.89894.log 2026-02-20T21:28:50.670 INFO:teuthology.orchestra.run.trial199.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.63262.log.gz 2026-02-20T21:28:50.670 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.81998.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55470.log 2026-02-20T21:28:50.670 INFO:teuthology.orchestra.run.trial199.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.81998.log.gz 2026-02-20T21:28:50.670 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.3.89894.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71222.log 2026-02-20T21:28:50.670 INFO:teuthology.orchestra.run.trial199.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.48498.log.gz 2026-02-20T21:28:50.671 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.55470.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.84866.log 2026-02-20T21:28:50.671 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.55470.log.gz 2026-02-20T21:28:50.671 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.71222.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59511.log 2026-02-20T21:28:50.671 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.1.84866.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.71222.log.gz 2026-02-20T21:28:50.671 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70364.log 2026-02-20T21:28:50.671 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.59511.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49660.log 2026-02-20T21:28:50.671 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.59511.log.gz 2026-02-20T21:28:50.671 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.70364.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.80121.log 2026-02-20T21:28:50.672 INFO:teuthology.orchestra.run.trial199.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.70364.log.gz 2026-02-20T21:28:50.672 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.49660.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83637.log 2026-02-20T21:28:50.672 INFO:teuthology.orchestra.run.trial199.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.49660.log.gz 2026-02-20T21:28:50.672 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.3.80121.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80574.log 2026-02-20T21:28:50.672 INFO:teuthology.orchestra.run.trial199.stderr: 19.1% -- replaced with /var/log/ceph/cluster1-client.mirror.3.80121.log.gz 2026-02-20T21:28:50.672 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.83637.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49431.log 2026-02-20T21:28:50.672 INFO:teuthology.orchestra.run.trial199.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.86760.log.gz 2026-02-20T21:28:50.672 INFO:teuthology.orchestra.run.trial199.stderr: 58.0% -- replaced with /var/log/ceph/cluster1-client.mirror.83637.log.gz 2026-02-20T21:28:50.672 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.80574.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82797.log 2026-02-20T21:28:50.672 INFO:teuthology.orchestra.run.trial199.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.3.89894.log.gz 2026-02-20T21:28:50.672 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.49431.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73417.log 2026-02-20T21:28:50.673 INFO:teuthology.orchestra.run.trial199.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.1.84866.log.gz 2026-02-20T21:28:50.673 INFO:teuthology.orchestra.run.trial199.stderr: 56.7% -- replaced with /var/log/ceph/cluster1-client.mirror.49431.log.gz 2026-02-20T21:28:50.673 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.82797.log: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.80574.log.gz 2026-02-20T21:28:50.673 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76684.log 2026-02-20T21:28:50.673 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.73417.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72409.log 2026-02-20T21:28:50.673 INFO:teuthology.orchestra.run.trial199.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.82797.log.gz 2026-02-20T21:28:50.673 INFO:teuthology.orchestra.run.trial199.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.73417.log.gz 2026-02-20T21:28:50.673 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76684.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60411.log 2026-02-20T21:28:50.673 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.2.log 2026-02-20T21:28:50.673 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.72409.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.76684.log.gz 2026-02-20T21:28:50.674 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.60411.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62851.log 2026-02-20T21:28:50.674 INFO:teuthology.orchestra.run.trial199.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.72409.log.gz 2026-02-20T21:28:50.674 INFO:teuthology.orchestra.run.trial199.stderr: 87.5%gzip -- replaced with /var/log/ceph/cluster1-client.admin.60411.log.gz -5/var/log/ceph/cluster2-osd.2.log: 2026-02-20T21:28:50.674 INFO:teuthology.orchestra.run.trial199.stderr: --verbose -- /var/log/ceph/cluster2-client.admin.46186.log 2026-02-20T21:28:50.674 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.62851.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74692.log 2026-02-20T21:28:50.674 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.46186.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.62851.log.gz 2026-02-20T21:28:50.674 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49519.log 2026-02-20T21:28:50.674 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46186.log.gz 2026-02-20T21:28:50.674 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.74692.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62341.log 2026-02-20T21:28:50.675 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.49519.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70259.log 2026-02-20T21:28:50.675 INFO:teuthology.orchestra.run.trial199.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74692.log.gz 2026-02-20T21:28:50.675 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.62341.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74202.log 2026-02-20T21:28:50.675 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.70259.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49519.log.gz 2026-02-20T21:28:50.675 INFO:teuthology.orchestra.run.trial199.stderr:gzip 87.3% -5 -- replaced with /var/log/ceph/cluster2-client.admin.62341.log.gz --verbose 2026-02-20T21:28:50.675 INFO:teuthology.orchestra.run.trial199.stderr: -- /var/log/ceph/cluster1-client.mirror.0.48489.log 2026-02-20T21:28:50.675 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.74202.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.70259.log.gz 2026-02-20T21:28:50.675 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68349.log 2026-02-20T21:28:50.675 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.0.48489.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.74202.log.gz 2026-02-20T21:28:50.675 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47964.log 2026-02-20T21:28:50.676 INFO:teuthology.orchestra.run.trial199.stderr: 19.1% -- replaced with /var/log/ceph/cluster1-client.mirror.0.48489.log.gz 2026-02-20T21:28:50.676 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.68349.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.90540.log 2026-02-20T21:28:50.676 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.47964.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.68349.log.gz 2026-02-20T21:28:50.676 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82441.log 2026-02-20T21:28:50.676 INFO:teuthology.orchestra.run.trial199.stderr: 63.6% -- replaced with /var/log/ceph/cluster1-client.mirror.47964.log.gz 2026-02-20T21:28:50.676 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.90540.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46435.log 2026-02-20T21:28:50.676 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.90540.log.gz 2026-02-20T21:28:50.676 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.82441.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76957.log 2026-02-20T21:28:50.676 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.46435.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.50736.log 2026-02-20T21:28:50.676 INFO:teuthology.orchestra.run.trial199.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46435.log.gz 2026-02-20T21:28:50.677 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.mirror.76957.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65710.log 2026-02-20T21:28:50.677 INFO:teuthology.orchestra.run.trial199.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82441.log.gz 2026-02-20T21:28:50.677 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.2.50736.log: 88.3%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.76957.log.gz 2026-02-20T21:28:50.677 INFO:teuthology.orchestra.run.trial199.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79416.log 2026-02-20T21:28:50.677 INFO:teuthology.orchestra.run.trial199.stderr: 19.7% -- replaced with /var/log/ceph/cluster1-client.mirror.2.50736.log.gz 2026-02-20T21:28:50.677 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster2-client.admin.65710.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72193.log 2026-02-20T21:28:50.677 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.79416.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.65710.log.gz 2026-02-20T21:28:50.677 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.78900.log 2026-02-20T21:28:50.678 INFO:teuthology.orchestra.run.trial199.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.79416.log.gz 2026-02-20T21:28:50.678 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.72193.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75083.log 2026-02-20T21:28:50.678 INFO:teuthology.orchestra.run.trial199.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.72193.log.gz 2026-02-20T21:28:50.678 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85745.log 2026-02-20T21:28:50.678 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.admin.78900.log: /var/log/ceph/cluster2-client.mirror.75083.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.78900.log.gz 2026-02-20T21:28:50.678 INFO:teuthology.orchestra.run.trial199.stderr: 80.6% -- replaced with /var/log/ceph/cluster2-client.mirror.75083.log.gz 2026-02-20T21:28:50.678 INFO:teuthology.orchestra.run.trial199.stderr:/var/log/ceph/cluster1-client.mirror.85745.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.85745.log.gz 2026-02-20T21:28:50.680 INFO:teuthology.orchestra.run.trial199.stderr: 94.8% -- replaced with /var/log/ceph/cluster2-client.mirror.0.54170.log.gz 2026-02-20T21:28:50.943 INFO:teuthology.orchestra.run.trial196.stderr: 91.1% -- replaced with /var/log/ceph/cluster1-mon.a.log.gz 2026-02-20T21:28:51.119 INFO:teuthology.orchestra.run.trial199.stderr: 91.1% -- replaced with /var/log/ceph/cluster2-mon.a.log.gz 2026-02-20T21:28:54.969 INFO:teuthology.orchestra.run.trial196.stderr: 94.5% -- replaced with /var/log/ceph/cluster1-osd.1.log.gz 2026-02-20T21:28:55.859 INFO:teuthology.orchestra.run.trial196.stderr: 94.3% -- replaced with /var/log/ceph/cluster1-osd.0.log.gz 2026-02-20T21:28:56.080 INFO:teuthology.orchestra.run.trial199.stderr: 94.5% -- replaced with /var/log/ceph/cluster2-osd.2.log.gz 2026-02-20T21:28:56.275 INFO:teuthology.orchestra.run.trial199.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-osd.1.log.gz 2026-02-20T21:28:56.574 INFO:teuthology.orchestra.run.trial196.stderr: 94.6% -- replaced with /var/log/ceph/cluster1-osd.2.log.gz 2026-02-20T21:28:56.576 INFO:teuthology.orchestra.run.trial196.stderr: 2026-02-20T21:28:56.576 INFO:teuthology.orchestra.run.trial196.stderr:real 0m6.294s 2026-02-20T21:28:56.576 INFO:teuthology.orchestra.run.trial196.stderr:user 0m16.615s 2026-02-20T21:28:56.577 INFO:teuthology.orchestra.run.trial196.stderr:sys 0m0.770s 2026-02-20T21:28:57.109 INFO:teuthology.orchestra.run.trial199.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-osd.0.log.gz 2026-02-20T21:28:57.111 INFO:teuthology.orchestra.run.trial199.stderr: 2026-02-20T21:28:57.112 INFO:teuthology.orchestra.run.trial199.stderr:real 0m6.808s 2026-02-20T21:28:57.112 INFO:teuthology.orchestra.run.trial199.stderr:user 0m19.173s 2026-02-20T21:28:57.112 INFO:teuthology.orchestra.run.trial199.stderr:sys 0m1.553s 2026-02-20T21:28:57.112 INFO:tasks.ceph:Archiving logs... 2026-02-20T21:28:57.113 DEBUG:teuthology.misc:Transferring archived files from trial196:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61415/remote/trial196/log 2026-02-20T21:28:57.114 DEBUG:teuthology.orchestra.run.trial196:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-20T21:28:58.394 DEBUG:teuthology.misc:Transferring archived files from trial199:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61415/remote/trial199/log 2026-02-20T21:28:58.395 DEBUG:teuthology.orchestra.run.trial199:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-20T21:29:00.404 DEBUG:teuthology.run_tasks:Unwinding manager install 2026-02-20T21:29:00.440 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2026-02-20T21:29:00.440 DEBUG:teuthology.orchestra.run.trial196:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-20T21:29:00.442 DEBUG:teuthology.orchestra.run.trial199:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-20T21:29:00.995 DEBUG:teuthology.run_tasks:Unwinding manager clock 2026-02-20T21:29:01.006 INFO:teuthology.task.clock:Checking final clock skew... 2026-02-20T21:29:01.007 DEBUG:teuthology.orchestra.run.trial196:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-20T21:29:01.009 DEBUG:teuthology.orchestra.run.trial199:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-20T21:29:01.023 INFO:teuthology.orchestra.run.trial196.stderr:bash: line 1: ntpq: command not found 2026-02-20T21:29:01.025 INFO:teuthology.orchestra.run.trial199.stderr:bash: line 1: ntpq: command not found 2026-02-20T21:29:01.049 INFO:teuthology.orchestra.run.trial196.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-20T21:29:01.049 INFO:teuthology.orchestra.run.trial196.stdout:=============================================================================== 2026-02-20T21:29:01.049 INFO:teuthology.orchestra.run.trial196.stdout:^- 139-177-202-26.ip.linode> 2 8 377 57 -9378us[-9378us] +/- 80ms 2026-02-20T21:29:01.049 INFO:teuthology.orchestra.run.trial196.stdout:^+ 45-79-189-79.ip.linodeus> 3 7 377 119 -137us[ -137us] +/- 30ms 2026-02-20T21:29:01.049 INFO:teuthology.orchestra.run.trial196.stdout:^* s2-a.time.mci1.us.rozint> 2 7 377 120 +334us[ +321us] +/- 23ms 2026-02-20T21:29:01.049 INFO:teuthology.orchestra.run.trial196.stdout:^+ 69-10-208-170.rainiercon> 2 7 377 378 -620us[ -650us] +/- 86ms 2026-02-20T21:29:01.050 INFO:teuthology.orchestra.run.trial199.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-20T21:29:01.050 INFO:teuthology.orchestra.run.trial199.stdout:=============================================================================== 2026-02-20T21:29:01.050 INFO:teuthology.orchestra.run.trial199.stdout:^- 139-177-202-26.ip.linode> 2 7 377 119 -10ms[ -10ms] +/- 82ms 2026-02-20T21:29:01.050 INFO:teuthology.orchestra.run.trial199.stdout:^+ 45-79-189-79.ip.linodeus> 3 6 377 55 -144us[ -144us] +/- 31ms 2026-02-20T21:29:01.050 INFO:teuthology.orchestra.run.trial199.stdout:^* s2-a.time.mci1.us.rozint> 2 7 377 123 +290us[ +283us] +/- 23ms 2026-02-20T21:29:01.050 INFO:teuthology.orchestra.run.trial199.stdout:^+ 69-10-208-170.rainiercon> 2 7 377 250 -402us[ -403us] +/- 63ms 2026-02-20T21:29:01.051 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2026-02-20T21:29:01.063 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2026-02-20T21:29:01.063 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2026-02-20T21:29:01.077 DEBUG:teuthology.orchestra.run.trial196:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2026-02-20T21:29:01.147 DEBUG:teuthology.orchestra.run.trial199:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2026-02-20T21:29:01.202 DEBUG:teuthology.orchestra.run.trial196:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2026-02-20T21:29:01.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T21:29:01.230 DEBUG:teuthology.orchestra.run.trial199:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2026-02-20T21:29:01.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T21:29:01.258 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2026-02-20T21:29:01.273 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2026-02-20T21:29:01.289 INFO:teuthology.task.internal:Duration was 1843.530285 seconds 2026-02-20T21:29:01.289 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2026-02-20T21:29:01.305 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2026-02-20T21:29:01.305 DEBUG:teuthology.orchestra.run.trial196:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-20T21:29:01.308 DEBUG:teuthology.orchestra.run.trial199:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-20T21:29:01.344 INFO:teuthology.orchestra.run.trial196.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-20T21:29:01.346 INFO:teuthology.orchestra.run.trial199.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-20T21:29:01.474 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2026-02-20T21:29:01.474 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial196.front.sepia.ceph.com 2026-02-20T21:29:01.475 DEBUG:teuthology.orchestra.run.trial196:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-20T21:29:01.493 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial199.front.sepia.ceph.com 2026-02-20T21:29:01.494 DEBUG:teuthology.orchestra.run.trial199:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-20T21:29:01.533 INFO:teuthology.task.internal.syslog:Gathering journactl... 2026-02-20T21:29:01.533 DEBUG:teuthology.orchestra.run.trial196:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-20T21:29:01.536 DEBUG:teuthology.orchestra.run.trial199:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-20T21:29:01.635 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2026-02-20T21:29:01.636 DEBUG:teuthology.orchestra.run.trial196:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-20T21:29:01.638 DEBUG:teuthology.orchestra.run.trial199:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-20T21:29:01.659 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-20T21:29:01.660 INFO:teuthology.orchestra.run.trial196.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-20T21:29:01.660 INFO:teuthology.orchestra.run.trial196.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-20T21:29:01.660 INFO:teuthology.orchestra.run.trial196.stderr: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-20T21:29:01.660 INFO:teuthology.orchestra.run.trial196.stderr:/home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-20T21:29:01.661 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-20T21:29:01.662 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-20T21:29:01.662 INFO:teuthology.orchestra.run.trial199.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-20T21:29:01.662 INFO:teuthology.orchestra.run.trial199.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-20T21:29:01.663 INFO:teuthology.orchestra.run.trial199.stderr:/home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-20T21:29:01.669 INFO:teuthology.orchestra.run.trial196.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: 82.3% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-20T21:29:01.672 INFO:teuthology.orchestra.run.trial199.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: 82.5% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-20T21:29:01.674 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2026-02-20T21:29:01.691 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2026-02-20T21:29:01.691 DEBUG:teuthology.orchestra.run.trial196:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-20T21:29:01.735 DEBUG:teuthology.orchestra.run.trial199:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-20T21:29:01.760 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2026-02-20T21:29:01.777 DEBUG:teuthology.orchestra.run.trial196:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-20T21:29:01.780 DEBUG:teuthology.orchestra.run.trial199:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-20T21:29:01.804 INFO:teuthology.orchestra.run.trial196.stdout:kernel.core_pattern = core 2026-02-20T21:29:01.823 INFO:teuthology.orchestra.run.trial199.stdout:kernel.core_pattern = core 2026-02-20T21:29:01.835 DEBUG:teuthology.orchestra.run.trial196:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-20T21:29:01.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T21:29:01.878 DEBUG:teuthology.orchestra.run.trial199:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-20T21:29:01.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T21:29:01.894 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2026-02-20T21:29:01.910 INFO:teuthology.task.internal:Transferring archived files... 2026-02-20T21:29:01.911 DEBUG:teuthology.misc:Transferring archived files from trial196:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61415/remote/trial196 2026-02-20T21:29:01.912 DEBUG:teuthology.orchestra.run.trial196:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-20T21:29:01.952 DEBUG:teuthology.misc:Transferring archived files from trial199:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61415/remote/trial199 2026-02-20T21:29:01.952 DEBUG:teuthology.orchestra.run.trial199:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-20T21:29:01.989 INFO:teuthology.task.internal:Removing archive directory... 2026-02-20T21:29:01.989 DEBUG:teuthology.orchestra.run.trial196:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-20T21:29:01.992 DEBUG:teuthology.orchestra.run.trial199:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-20T21:29:02.039 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2026-02-20T21:29:02.057 INFO:teuthology.task.internal:Not uploading archives. 2026-02-20T21:29:02.057 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2026-02-20T21:29:02.074 INFO:teuthology.task.internal:Tidying up after the test... 2026-02-20T21:29:02.074 DEBUG:teuthology.orchestra.run.trial196:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-20T21:29:02.077 DEBUG:teuthology.orchestra.run.trial199:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-20T21:29:02.092 INFO:teuthology.orchestra.run.trial196.stdout: 917572 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 20 21:29 /home/ubuntu/cephtest 2026-02-20T21:29:02.094 INFO:teuthology.orchestra.run.trial199.stdout: 917521 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 20 21:29 /home/ubuntu/cephtest 2026-02-20T21:29:02.096 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2026-02-20T21:29:02.112 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2026-02-20T21:29:02.151 INFO:teuthology.run:Summary data: description: rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-snappy supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-workunit-fast-diff} duration: 1843.5302846431732 flavor: default owner: scheduled_yuriw@soko04.front.sepia.ceph.com success: true 2026-02-20T21:29:02.152 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-20T21:29:02.268 INFO:teuthology.run:pass